CE-882 - Disable a failing test...

Feature/quartz schedule new jobs

.circleci/config.yml

@@ -98,6 +98,31 @@ commands:
             - ~/.m2
           key: v1-dependencies-{{ checksum "pom.xml" }}
 
+  install_asciidoctor:
+    steps:
+      - checkout
+      - run:
+          name: Install asciidoctor
+          command: |
+            sudo apt-get update
+            sudo apt install -y asciidoctor
+
+  run_asciidoctor:
+    steps:
+      - run:
+          name: Run asciidoctor
+          command: |
+            cd docs
+            asciidoctor -a docinfo=shared index.adoc
+
+  upload_docs_site:
+    steps:
+      - run:
+          name: scp html to justinsgotskinnylegs.com
+          command: |
+            cd docs
+            scp index.html dkelkhoff@45.79.44.221:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html
+
 jobs:
   mvn_test:
     executor: localstack/default
@@ -114,6 +139,13 @@ jobs:
       - mvn_verify
       - mvn_jar_deploy
 
+  publish_asciidoc:
+    executor: localstack/default
+    steps:
+      - install_asciidoctor
+      - run_asciidoctor
+      - upload_docs_site
+
 workflows:
   test_only:
     jobs:
@@ -121,7 +153,7 @@ workflows:
           context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
           filters:
             branches:
-              ignore: /dev/
+              ignore: /(dev|integration.*)/
             tags:
               ignore: /(version|snapshot)-.*/
 
@@ -131,7 +163,10 @@ workflows:
           context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
           filters:
             branches:
-              only: /dev/
+              only: /(dev|integration.*)/
             tags:
               only: /(version|snapshot)-.*/
-
+      - publish_asciidoc:
+          filters:
+            branches:
+              only: /dev/

docs/metaData/Fields.adoc

@@ -21,5 +21,99 @@ Used to set values in the `displayValues` map within a `QRecord`.
 * `possibleValueSourceName` - *String* - Reference to a {link-pvs} to be used for this field.
 Values in this field should correspond to ids from the referenced Possible Value Source.
 * `maxLength` - *Integer* - Maximum length (number of characters) allowed for values in this field.
-Only applicable for fields with `type=STRING`.
-* `
+Only applicable for fields with `type=STRING`.  Needs to be used with a `FieldBehavior` of type `ValueTooLongBehavior`.
+
+==== Field Behaviors
+Additional behaviors can be attached to fields through the use of the `behaviors` attribute,
+which is a `Set` of 0 or more instances of implementations of the `FieldBehavior` interface.
+Note that in some cases, these instances may be `enum` constants,
+but other times may be regular Objects.
+
+QQQ provides a set of common field behaviors.
+Applications can also define their own field behaviors by implementing the `FieldBehavior` interface,
+and attaching instances of their custom behavior classes to fields.
+
+===== ValueTooLongBehavior
+Used on String fields.  Requires the field to have a `maxLength` set.
+Depending on the chosen instance of this enum, before a record is Inserted or Updated,
+if the value in the field is longer than the `maxLength`, then one of the following actions can occur:
+
+* `TRUNCATE` - The value will be simply truncated to the `maxLength`.
+* `TRUNCATE_ELLIPSIS` - The value will be truncated to 3 characters less than the `maxLength`, and three periods (an ellipsis) will be placed at the end.
+* `ERROR` - An error will be reported, and the record will not be inserted or updated.
+* `PASS_THROUGH` - Nothing will happen.  This is the same as not having a `ValueTooLongBehavior` on the field.
+
+[source,java]
+.Examples of using ValueTooLongBehavior
+----
+   new QFieldMetaData("sku", QFieldType.STRING)
+      .withMaxLength(40),
+      .withBehavior(ValueTooLongBehavior.ERROR),
+
+   new QFieldMetaData("reason", QFieldType.STRING)
+      .withMaxLength(250),
+      .withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS),
+
+----
+
+===== DynamicDefaultValueBehavior
+Used to set a dynamic default value to a field when it is being inserted or updated.
+For example, instead of having a hard-coded `defaultValue` specified in the field meta-data,
+and instead of having to add, for example, a pre-insert custom action.
+
+* `CREATE_DATE` - On inserts, sets the field's value to the current time.
+* `MODIFY_DATE` - On inserts and updates, sets the field's value to the current time.
+* `USER_ID` - On inserts and updates, sets the field's value to the current user's id (but only if the value is currently null).
+
+_Note that the `QInstanceEnricher` will, by default, add the `CREATE_DATE` and `MODIFY_DATE` `DynamicDefaultValueBehavior`
+options to any fields named `"createDate"` or `"modifyDate"`.
+This behavior can be disabled by setting the `configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate` property
+on the `QInstanceEnricher` instance used by the application to `false`._
+
+[source,java]
+.Examples of using DynamicDefaultValueBehavior
+----
+   new QFieldMetaData("createDate", QFieldType.DATE_TIME)
+      .withBehavior(DynamicDefaultValueBehavior.CREATE_DATE),
+
+   new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)
+      .withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE),
+
+   new QFieldMetaData("createdByUserId", QFieldType.STRING)
+      .withBehavior(DynamicDefaultValueBehavior.USER_ID),
+----
+
+===== DateTimeDisplayValueBehavior
+By default, in QQQ, fields of type `DATE_TIME` are stored in UTC,
+and their values in a QRecord is a java `Instant` instance, which is always UTC.
+However, frontends will prefer to display date-time values in the user's local Time Zone whenever possible.
+
+Using `DateTimeDisplayValueBehavior` allows a `DATE_TIME` field to be displayed in a different Time Zone.
+An example use-case for this would be displaying airplane flight times,
+where you would want a flight from California to New York to display Pacific Time for its departure time,
+and Eastern Time for its arrival.
+
+An instance of `DateTimeDisplayValueBehavior` can be configured to either use a hard-coded time `ZoneId`
+(for example, to always show users UTC, or a business's home-office time zone).
+Or, it can be set up to get the time zone to use from another field in the table.
+
+[source,java]
+.Examples of using DateTimeDisplayValueBehavior
+----
+new QTableMetaData().withName("flights").withFields(List.of(
+   ...
+   new QFieldMetaData("departureTimeZoneId", QFieldType.STRING),
+   new QFieldMetaData("arrivaTimeZoneId", QFieldType.STRING),
+
+   new QFieldMetaData("departureTime", QFieldType.DATE_TIME)
+      .withBehavior(new DateTimeDisplayValueBehavior()
+         .withZoneIdFromFieldName("departureTimeZoneId")),
+
+   new QFieldMetaData("arrivalTime", QFieldType.DATE_TIME)
+      .withBehavior(new DateTimeDisplayValueBehavior()
+         .withZoneIdFromFieldName("arrivalTimeZoneId"))
+
+   new QFieldMetaData("ticketSaleStartDateTime", QFieldType.DATE_TIME)
+      .withBehavior(new DateTimeDisplayValueBehavior()
+         .withDefaultZoneId("UTC"))
+----

pom.xml

@@ -245,8 +245,7 @@ echo "   See also target/site/jacoco/index.html"
 echo "   and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
 echo "------------------------------------------------------------"
 
-which xpath > /dev/null 2>&1
-if [ "$?" == "0" ]; then
+if which xpath > /dev/null 2>&1; then
    echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
    xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
    paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
@@ -255,8 +254,7 @@ else
    echo "xpath is not installed.  Jacoco coverage summary will not be produced here...";
 fi
 
-which xpath > /dev/null 2>&1
-if [ "$?" == "0" ]; then
+if which html2text > /dev/null 2>&1; then
    echo "Untested classes, per Jacoco:"
    echo "-----------------------------"
    for i in target/site/jacoco/*/index.html; do

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java

@@ -31,6 +31,7 @@ import java.io.Serializable;
  *******************************************************************************/
 public class AsyncJobStatus implements Serializable
 {
+   private String        jobName;
    private AsyncJobState state;
    private String        message;
    private Integer       current;
@@ -187,4 +188,36 @@ public class AsyncJobStatus implements Serializable
    {
       this.cancelRequested = cancelRequested;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for jobName
+    *******************************************************************************/
+   public String getJobName()
+   {
+      return (this.jobName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for jobName
+    *******************************************************************************/
+   public void setJobName(String jobName)
+   {
+      this.jobName = jobName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for jobName
+    *******************************************************************************/
+   public AsyncJobStatus withJobName(String jobName)
+   {
+      this.jobName = jobName;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/NonPersistedAsyncJobCallback.java

@@ -0,0 +1,62 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.async;
+
+
+import java.util.UUID;
+
+
+/*******************************************************************************
+ ** subclass designed to be used when we want there to be an instance (so code
+ ** doesn't have to all be null-tolerant), but there's no one who will ever be
+ ** reading the status data, so we don't need to store the object in a
+ ** state provider.
+ *******************************************************************************/
+public class NonPersistedAsyncJobCallback extends AsyncJobCallback
+{
+   private final AsyncJobStatus asyncJobStatus;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public NonPersistedAsyncJobCallback(UUID jobUUID, AsyncJobStatus asyncJobStatus)
+   {
+      super(jobUUID, asyncJobStatus);
+      this.asyncJobStatus = asyncJobStatus;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   protected void storeUpdatedStatus()
+   {
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // noop - cf. base class, which writes to persistence here (our point is, we do not) //
+      ///////////////////////////////////////////////////////////////////////////////////////
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -149,7 +149,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             // sort the field names by their labels //
             //////////////////////////////////////////
             List<String> sortedFieldNames = table.getFields().keySet().stream()
-               .sorted(Comparator.comparing(fieldName -> table.getFields().get(fieldName).getLabel()))
+               .sorted(Comparator.comparing(fieldName -> Objects.requireNonNullElse(table.getFields().get(fieldName).getLabel(), fieldName)))
                .toList();
 
             QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
@@ -303,7 +303,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
          }
          else
          {
-            String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
+            String formattedValue = getFormattedValueForAuditDetail(table, record, fieldName, field, value);
             detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
             detailRecord.withValue("newValue", formattedValue);
          }
@@ -329,8 +329,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             }
             else
             {
-               String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
-               String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
+               String formattedValue    = getFormattedValueForAuditDetail(table, record, fieldName, field, value);
+               String formattedOldValue = getFormattedValueForAuditDetail(table, oldRecord, fieldName, field, oldValue);
 
                if(oldValue == null)
                {
@@ -464,7 +464,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static String getFormattedValueForAuditDetail(QRecord record, String fieldName, QFieldMetaData field, Serializable value)
+   private static String getFormattedValueForAuditDetail(QTableMetaData table, QRecord record, String fieldName, QFieldMetaData field, Serializable value)
    {
       String formattedValue = null;
       if(value != null)
@@ -479,7 +479,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
          }
          else
          {
-            formattedValue = QValueFormatter.formatValue(field, value);
+            QValueFormatter.setDisplayValuesInRecord(table, table.getFields(), record);
+            formattedValue = record.getDisplayValue(fieldName);
          }
       }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -526,7 +526,7 @@ public class PollingAutomationPerTableRunner implements Runnable
          // note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
          //////////////////////////////////////////////////////////////////////////////////////////////////////////////
          List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
-         LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
+         LOG.debug("Of the [" + records.size() + "] records that were pending automations, [" + matchingQRecords.size() + "] of them match the filter on the action:" +  action);
          if(CollectionUtils.nullSafeHasContents(matchingQRecords))
          {
             LOG.debug("  Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
@@ -601,7 +601,7 @@ public class PollingAutomationPerTableRunner implements Runnable
 
    /*******************************************************************************
     ** Finally, actually run action code against a list of known matching records.
-    ** todo not commit - move to somewhere genericer
+    **
     *******************************************************************************/
    public static void applyActionToMatchingRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action) throws Exception
    {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java

@@ -96,7 +96,7 @@ public class QCodeLoader
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //
@@ -135,7 +135,7 @@ public class QCodeLoader
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //
@@ -187,7 +187,7 @@ public class QCodeLoader
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/AbstractWidgetRenderer.java

@@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -72,80 +73,104 @@ public abstract class AbstractWidgetRenderer
     *******************************************************************************/
    protected boolean setupDropdowns(RenderWidgetInput input, QWidgetMetaData metaData, QWidgetData widgetData) throws QException
    {
-      List<List<Map<String, String>>> pvsData                   = new ArrayList<>();
-      List<String>                    pvsLabels                 = new ArrayList<>();
-      List<String>                    pvsNames                  = new ArrayList<>();
+      List<List<Map<String, String>>> dataList                  = new ArrayList<>();
+      List<String>                    labelList                 = new ArrayList<>();
+      List<String>                    nameList                  = new ArrayList<>();
       List<String>                    missingRequiredSelections = new ArrayList<>();
       for(WidgetDropdownData dropdownData : CollectionUtils.nonNullList(metaData.getDropdowns()))
       {
-         String               possibleValueSourceName = dropdownData.getPossibleValueSourceName();
-         QPossibleValueSource possibleValueSource     = input.getInstance().getPossibleValueSource(possibleValueSourceName);
-
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // this looks complicated, but is just look for a label in the dropdown data and if found use it,                                                                                  //
-         // otherwise look for label in PVS and if found use that, otherwise just use the PVS name                                                                                          //
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName);
-         pvsLabels.add(pvsLabel);
-         pvsNames.add(possibleValueSourceName);
-
-         SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput();
-         pvsInput.setPossibleValueSourceName(possibleValueSourceName);
-
-         if(dropdownData.getForeignKeyFieldName() != null)
+         if(WidgetDropdownType.DATE_PICKER.equals(dropdownData.getType()))
          {
-            ////////////////////////////////////////
-            // look for an id in the query params //
-            ////////////////////////////////////////
-            Integer id = null;
-            if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id")))
+            String name = dropdownData.getName();
+            nameList.add(name);
+            labelList.add(dropdownData.getLabel());
+            dataList.add(new ArrayList<>());
+
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // sure that something has been selected, and if not, display a message that a selection needs made       //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(dropdownData.getIsRequired())
             {
-               id = Integer.parseInt(input.getQueryParams().get("id"));
-            }
-            if(id != null)
-            {
-               pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria(
-                  new QFilterCriteria(
-                     dropdownData.getForeignKeyFieldName(),
-                     QCriteriaOperator.EQUALS,
-                     id)));
+               if(!input.getQueryParams().containsKey(name) || !StringUtils.hasContent(input.getQueryParams().get(name)))
+               {
+                  missingRequiredSelections.add(dropdownData.getLabel());
+               }
             }
          }
-
-         SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput);
-
-         List<Map<String, String>> dropdownOptionList = new ArrayList<>();
-         pvsData.add(dropdownOptionList);
-
-         //////////////////////////////////////////
-         // sort results, dedupe, and add to map //
-         //////////////////////////////////////////
-         Set<String> exists = new HashSet<>();
-         output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel()));
-         for(QPossibleValue<?> possibleValue : output.getResults())
+         else
          {
-            dropdownOptionList.add(MapBuilder.of(
-               "id", String.valueOf(possibleValue.getId()),
-               "label", possibleValue.getLabel()
-            ));
-         }
-
-         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // because we know the dropdowns and what the field names will be when something is selected, we can make //
-         // sure that something has been selected, and if not, display a message that a selection needs made       //
-         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         if(dropdownData.getIsRequired())
-         {
-            if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName)))
+            String possibleValueSourceName = dropdownData.getPossibleValueSourceName();
+            if(possibleValueSourceName != null)
             {
-               missingRequiredSelections.add(pvsLabel);
+               QPossibleValueSource possibleValueSource = input.getInstance().getPossibleValueSource(possibleValueSourceName);
+
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // this looks complicated, but is just look for a label in the dropdown data and if found use it,                                                                                  //
+               // otherwise look for label in PVS and if found use that, otherwise just use the PVS name                                                                                          //
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName);
+               labelList.add(pvsLabel);
+               nameList.add(possibleValueSourceName);
+
+               SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput();
+               pvsInput.setPossibleValueSourceName(possibleValueSourceName);
+
+               if(dropdownData.getForeignKeyFieldName() != null)
+               {
+                  ////////////////////////////////////////
+                  // look for an id in the query params //
+                  ////////////////////////////////////////
+                  Integer id = null;
+                  if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id")))
+                  {
+                     id = Integer.parseInt(input.getQueryParams().get("id"));
+                  }
+                  if(id != null)
+                  {
+                     pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria(
+                        new QFilterCriteria(
+                           dropdownData.getForeignKeyFieldName(),
+                           QCriteriaOperator.EQUALS,
+                           id)));
+                  }
+               }
+
+               SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput);
+
+               List<Map<String, String>> dropdownOptionList = new ArrayList<>();
+               dataList.add(dropdownOptionList);
+
+               //////////////////////////////////////////
+               // sort results, dedupe, and add to map //
+               //////////////////////////////////////////
+               Set<String> exists = new HashSet<>();
+               output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel()));
+               for(QPossibleValue<?> possibleValue : output.getResults())
+               {
+                  dropdownOptionList.add(MapBuilder.of(
+                     "id", String.valueOf(possibleValue.getId()),
+                     "label", possibleValue.getLabel()
+                  ));
+               }
+
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // because we know the dropdowns and what the field names will be when something is selected, we can make //
+               // sure that something has been selected, and if not, display a message that a selection needs made       //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(dropdownData.getIsRequired())
+               {
+                  if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName)))
+                  {
+                     missingRequiredSelections.add(pvsLabel);
+                  }
+               }
             }
          }
       }
 
-      widgetData.setDropdownNameList(pvsNames);
-      widgetData.setDropdownLabelList(pvsLabels);
-      widgetData.setDropdownDataList(pvsData);
+      widgetData.setDropdownNameList(nameList);
+      widgetData.setDropdownLabelList(labelList);
+      widgetData.setDropdownDataList(dataList);
 
       ////////////////////////////////////////////////////////////////////////////////
       // if there are any missing required dropdowns, build up a message to display //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/DistinctFilteringRecordPipe.java

@@ -0,0 +1,146 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+
+
+/*******************************************************************************
+ ** Subclass of record pipe that ony allows through distinct records, based on
+ ** the set of fields specified in the constructor as a uniqueKey.
+ *******************************************************************************/
+public class DistinctFilteringRecordPipe extends RecordPipe
+{
+   private UniqueKey         uniqueKey;
+   private Set<Serializable> seenValues = new HashSet<>();
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public DistinctFilteringRecordPipe(UniqueKey uniqueKey)
+   {
+      this.uniqueKey = uniqueKey;
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor that accepts pipe's overrideCapacity (allowed to be null)
+    **
+    *******************************************************************************/
+   public DistinctFilteringRecordPipe(UniqueKey uniqueKey, Integer overrideCapacity)
+   {
+      super(overrideCapacity);
+      this.uniqueKey = uniqueKey;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addRecords(List<QRecord> records) throws QException
+   {
+      List<QRecord> recordsToAdd = new ArrayList<>();
+      for(QRecord record : records)
+      {
+         if(!seenBefore(record))
+         {
+            recordsToAdd.add(record);
+         }
+      }
+
+      if(recordsToAdd.isEmpty())
+      {
+         return;
+      }
+
+      super.addRecords(recordsToAdd);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addRecord(QRecord record) throws QException
+   {
+      if(seenBefore(record))
+      {
+         return;
+      }
+
+      super.addRecord(record);
+   }
+
+
+
+   /*******************************************************************************
+    ** return true if we've seen this record before (based on the unique key) -
+    ** also - update the set of seen values!
+    *******************************************************************************/
+   private boolean seenBefore(QRecord record)
+   {
+      Serializable ukValues = extractUKValues(record);
+      if(seenValues.contains(ukValues))
+      {
+         return true;
+      }
+      seenValues.add(ukValues);
+      return false;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Serializable extractUKValues(QRecord record)
+   {
+      if(uniqueKey.getFieldNames().size() == 1)
+      {
+         return (record.getValue(uniqueKey.getFieldNames().get(0)));
+      }
+      else
+      {
+         ArrayList<Serializable> rs = new ArrayList<>();
+         for(String fieldName : uniqueKey.getFieldNames())
+         {
+            rs.add(record.getValue(fieldName));
+         }
+         return (rs);
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java

@@ -190,6 +190,9 @@ public class ExportAction
       Set<String>     addedJoinNames = new HashSet<>();
       if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames()))
       {
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         // make sure that any tables being selected from are included as (LEFT) joins in the query //
+         /////////////////////////////////////////////////////////////////////////////////////////////
          for(String fieldName : exportInput.getFieldNames())
          {
             if(fieldName.contains("."))
@@ -198,27 +201,7 @@ public class ExportAction
                String   joinTableName = parts[0];
                if(!addedJoinNames.contains(joinTableName))
                {
-                  QueryJoin queryJoin = new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true);
-                  queryJoins.add(queryJoin);
-
-                  /////////////////////////////////////////////////////////////////////////////////////////////
-                  // in at least some cases, we need to let the queryJoin know what join-meta-data to use... //
-                  // This code basically mirrors what QFMD is doing right now, so it's better -              //
-                  // but shouldn't all of this just be in JoinsContext?  it does some of this...             //
-                  /////////////////////////////////////////////////////////////////////////////////////////////
-                  QTableMetaData        table               = exportInput.getTable();
-                  Optional<ExposedJoin> exposedJoinOptional = CollectionUtils.nonNullList(table.getExposedJoins()).stream().filter(ej -> ej.getJoinTable().equals(joinTableName)).findFirst();
-                  if(exposedJoinOptional.isEmpty())
-                  {
-                     throw (new QException("Could not find exposed join between base table " + table.getName() + " and requested join table " + joinTableName));
-                  }
-                  ExposedJoin exposedJoin = exposedJoinOptional.get();
-
-                  if(exposedJoin.getJoinPath().size() == 1)
-                  {
-                     queryJoin.setJoinMetaData(QContext.getQInstance().getJoin(exposedJoin.getJoinPath().get(exposedJoin.getJoinPath().size() - 1)));
-                  }
-
+                  queryJoins.add(new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true));
                   addedJoinNames.add(joinTableName);
                }
             }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -136,8 +136,8 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
     *******************************************************************************/
    public ReportOutput execute(ReportInput reportInput) throws QException
    {
-      ReportOutput reportOutput = new ReportOutput();
-      QReportMetaData report = getReportMetaData(reportInput);
+      ReportOutput    reportOutput = new ReportOutput();
+      QReportMetaData report       = getReportMetaData(reportInput);
 
       this.views = report.getViews();
       this.dataSources = report.getDataSources();
@@ -398,7 +398,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       RunBackendStepInput   finalTransformStepInput  = transformStepInput;
       RunBackendStepOutput  finalTransformStepOutput = transformStepOutput;
 
-      String tableLabel = ObjectUtils.tryElse(() -> QContext.getQInstance().getTable(dataSource.getSourceTable()).getLabel(), Objects.requireNonNullElse(dataSource.getSourceTable(), ""));
+      String        tableLabel    = ObjectUtils.tryElse(() -> QContext.getQInstance().getTable(dataSource.getSourceTable()).getLabel(), Objects.requireNonNullElse(dataSource.getSourceTable(), ""));
       AtomicInteger consumedCount = new AtomicInteger(0);
 
       /////////////////////////////////////////////////////////////////
@@ -531,7 +531,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private FieldAndJoinTable getFieldAndJoinTable(QTableMetaData mainTable, String fieldName) throws QException
+   public static FieldAndJoinTable getFieldAndJoinTable(QTableMetaData mainTable, String fieldName) throws QException
    {
       if(fieldName.indexOf('.') > -1)
       {
@@ -704,8 +704,24 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       //////////////////////////////////////////////////////////////////////////////////////
       for(String fieldName : record.getValues().keySet())
       {
-         FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, fieldName);
-         QFieldMetaData    field             = fieldAndJoinTable.field();
+         QFieldMetaData field = null;
+         try
+         {
+            //////////////////////////////////////////////////////
+            // todo - memoize this, if we ever need to optimize //
+            //////////////////////////////////////////////////////
+            FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, fieldName);
+            field = fieldAndJoinTable.field();
+         }
+         catch(Exception e)
+         {
+            //////////////////////////////////////////////////////////////////////////////////////
+            // for non-real-fields... let's skip for now - but maybe treat as string in future? //
+            //////////////////////////////////////////////////////////////////////////////////////
+            LOG.debug("Couldn't find field in table qInstance - won't compute aggregates for it", logPair("fieldName", fieldName));
+            continue;
+         }
+
          if(StringUtils.hasContent(field.getPossibleValueSourceName()))
          {
             @SuppressWarnings("unchecked")
@@ -1097,5 +1113,22 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable) {}
+   public record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable)
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public String getLabel(QTableMetaData mainTable)
+      {
+         if(mainTable.getName().equals(joinTable.getName()))
+         {
+            return (field.getLabel());
+         }
+         else
+         {
+            return (joinTable.getLabel() + ": " + field.getLabel());
+         }
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -43,8 +44,9 @@ public class RecordPipe
 
    private static final long BLOCKING_SLEEP_MILLIS = 100;
    private static final long MAX_SLEEP_LOOP_MILLIS = 300_000; // 5 minutes
+   private static final int  DEFAULT_CAPACITY      = 1_000;
 
-   private int capacity = 1_000;
+   private int capacity = DEFAULT_CAPACITY;
    private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(capacity);
 
    private boolean isTerminated = false;
@@ -70,11 +72,13 @@ public class RecordPipe
 
    /*******************************************************************************
     ** Construct a record pipe, with an alternative capacity for the internal queue.
+    **
+    ** overrideCapacity is allowed to be null - in which case, DEFAULT_CAPACITY is used.
     *******************************************************************************/
    public RecordPipe(Integer overrideCapacity)
    {
-      this.capacity = overrideCapacity;
-      queue = new ArrayBlockingQueue<>(overrideCapacity);
+      this.capacity = Objects.requireNonNullElse(overrideCapacity, DEFAULT_CAPACITY);
+      queue = new ArrayBlockingQueue<>(this.capacity);
    }
 
 
@@ -138,7 +142,7 @@ public class RecordPipe
          {
             if(now - sleepLoopStartTime > MAX_SLEEP_LOOP_MILLIS)
             {
-               LOG.warn("Giving up adding record to pipe, due to pipe being full for more than {} millis", MAX_SLEEP_LOOP_MILLIS);
+               LOG.warn("Giving up adding record to pipe, due to pipe being full for more than " + MAX_SLEEP_LOOP_MILLIS + " millis");
                throw (new IllegalStateException("Giving up adding record to pipe, due to pipe staying full too long."));
             }
             LOG.trace("Record pipe.add failed (due to full pipe).  Blocking.");

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java

@@ -82,12 +82,26 @@ import org.apache.poi.xssf.usermodel.XSSFWorkbook;
 /*******************************************************************************
  ** Excel export format implementation using POI library, but with modifications
  ** to actually stream output rather than use any temp files.
+ **
+ ** For a rough outline:
+ ** - create a basically empty Excel workbook using POI - empty meaning, without
+ ** data rows.
+ ** - have POI write that workbook out into a byte[] - which will be a zip full
+ ** of xml (e.g., xlsx).
+ ** - then open a new ZipOutputStream wrapper around the OutputStream we took in
+ ** as the report destination (e.g., streamed into storage or http output)
+ ** - Copy over all entries from the xlsx into our new zip-output-stream, other than
+ ** ones that are the actual sheets that we want to put data into.
+ ** - For the sheet entries, use the StreamedPoiSheetWriter class to write the
+ ** report's data directly out as Excel XML (not using POI).
+ ** - Pivot tables require a bit of an additional hack - to write a "pivot cache
+ ** definition", which, while we won't put all the data in it (because we'll tell
+ ** it refreshOnLoad="true"), we will at least need to know how many cols & rows
+ ** are in the data-sheet (which we wouldn't know until we streamed that sheet!)
  *******************************************************************************/
 public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInterface
 {
    private static final QLogger LOG = QLogger.getLogger(ExcelPoiBasedStreamingExportStreamer.class);
-   public static final String EXCEL_DATE_FORMAT = "yyyy-MM-dd";
-   public static final String EXCEL_DATE_TIME_FORMAT = "yyyy-MM-dd H:mm:ss";
 
    private List<QReportView>    views;
    private ExportInput          exportInput;
@@ -95,17 +109,20 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
    private OutputStream         outputStream;
    private ZipOutputStream      zipOutputStream;
 
-   private PoiExcelStylerInterface poiExcelStylerInterface = getStylerInterface();
-   private Map<String, String>     excelCellFormats;
+   public static final String EXCEL_DATE_FORMAT      = "yyyy-MM-dd";
+   public static final String EXCEL_DATE_TIME_FORMAT = "yyyy-MM-dd H:mm:ss";
+
+   private PoiExcelStylerInterface    poiExcelStylerInterface = getStylerInterface();
+   private Map<String, String>        excelCellFormats;
+   private Map<String, XSSFCellStyle> styles                  = new HashMap<>();
 
    private int rowNo      = 0;
    private int sheetIndex = 1;
 
-   private Map<String, String>        pivotViewToCacheDefinitionReferenceMap = new HashMap<>();
-   private Map<String, XSSFCellStyle> styles                                 = new HashMap<>();
+   private Map<String, String> pivotViewToCacheDefinitionReferenceMap = new HashMap<>();
 
-   private Writer                 activeSheetWriter = null;
-   private StreamedPoiSheetWriter sheetWriter       = null;
+   private Writer              activeSheetWriter = null;
+   private StreamedSheetWriter sheetWriter       = null;
 
    private QReportView                       currentView      = null;
    private Map<String, List<QFieldMetaData>> fieldsPerView    = new HashMap<>();
@@ -194,6 +211,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
 
          /////////////////////////////////////////////////////////////////////////////////////////////
          // open up a zipOutputStream around the output stream that the report is to be written to. //
+         // note:  this stream is closed in the finish method - see more comments there.            //
          /////////////////////////////////////////////////////////////////////////////////////////////
          this.zipOutputStream = new ZipOutputStream(this.outputStream);
 
@@ -271,7 +289,6 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
       for(QReportField column : dataView.getColumns())
       {
          XSSFCell cell = headerRow.createCell(columnNo++);
-         // todo ... not like this
          cell.setCellValue(QInstanceEnricher.nameToLabel(column.getName()));
       }
 
@@ -313,10 +330,6 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
       {
          int columnLabelColumnIndex = getColumnIndex(dataView.getColumns(), value.getFieldName());
 
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // todo - some bug where, if use a group-by field here, then ... it doesn't get used for the grouping. //
-         //  g-sheets does let me do this, so, maybe, download their file and see how it's different?           //
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////
          String labelPrefix = value.getFunction().name() + " of ";
          String label       = labelPrefix + QInstanceEnricher.nameToLabel(value.getFieldName());
          String valueFormat = null;
@@ -435,7 +448,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
          //////////////////////////////////////////
          zipOutputStream.putNextEntry(new ZipEntry("xl/worksheets/sheet" + this.sheetIndex++ + ".xml"));
          activeSheetWriter = new OutputStreamWriter(zipOutputStream);
-         sheetWriter = new StreamedPoiSheetWriter(activeSheetWriter);
+         sheetWriter = new StreamedSheetWriter(activeSheetWriter);
 
          if(ReportType.PIVOT.equals(view.getType()))
          {
@@ -582,7 +595,13 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
                   String format = excelCellFormats.get(field.getName());
                   if(format != null)
                   {
-                     // todo - formats...
+                     ////////////////////////////////////////////////////////////////////////////////////////////
+                     // todo - so - for this streamed/zip approach, we need to know all styles before we start //
+                     // any sheets.  but, right now Report action only calls us with per-sheet styles when     //
+                     // it's starting individual sheets.  so, we can't quite support this at this time.        //
+                     // "just" need to change GenerateReportAction to look up all cell formats for all sheets  //
+                     // before preRun is called... and change all existing streamer classes to handle that too //
+                     ////////////////////////////////////////////////////////////////////////////////////////////
                      // worksheet.style(rowNo, col).format(format).set();
                   }
                }
@@ -609,7 +628,6 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
             }
             else if(value instanceof Instant i)
             {
-               // todo - what would be a better zone to use here?
                sheetWriter.createCell(col, DateUtil.getExcelDate(i.atZone(ZoneId.systemDefault()).toLocalDateTime()), dateTimeStyleIndex);
             }
             else
@@ -656,10 +674,11 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
          //////////////////////////////////////////////
          closeLastSheetIfOpen();
 
-         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // note - leave the zipOutputStream open.  It is a wrapper around the OutputStream we were given by the caller, //
-         // so it is their responsibility to close that stream (which implicitly closes the zip, it appears)             //
-         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         /////////////////////////////////////////////////////////////////////////////////////
+         // so, we DO need a close here, on the zipOutputStream, to finish its "zippiness". //
+         // even though, doing so also closes the outputStream from the caller that this    //
+         // zipOutputStream is wrapping (and the caller will likely call close on too)...   //
+         /////////////////////////////////////////////////////////////////////////////////////
          zipOutputStream.close();
       }
       catch(Exception e)
@@ -748,7 +767,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
          zipOutputStream.putNextEntry(new ZipEntry(pivotViewToCacheDefinitionReferenceMap.get(pivotTableView.getName())));
 
          /////////////////////////////////////////////////////////
-         // prepare the xml for each field (e.g., w/ its labelO //
+         // prepare the xml for each field (e.g., w/ its label) //
          /////////////////////////////////////////////////////////
          List<String> cachedFieldElements = new ArrayList<>();
          for(QFieldMetaData column : this.fieldsPerView.get(dataView.getName()))
@@ -766,7 +785,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
          activeSheetWriter = new OutputStreamWriter(zipOutputStream);
          activeSheetWriter.write(String.format("""
                <?xml version="1.0" encoding="UTF-8"?>
-               <pivotCacheDefinition xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" createdVersion="3" minRefreshableVersion="3" refreshedVersion="3" refreshedBy="Apache POI" refreshedDate="1.7113>  95767702E12" refreshOnLoad="true" r:id="rId1">
+               <pivotCacheDefinition xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" createdVersion="3" minRefreshableVersion="3" refreshedVersion="3" refreshedBy="Apache POI" refreshedDate="1.711395767702E12" refreshOnLoad="true" r:id="rId1">
                  <cacheSource type="worksheet">
                    <worksheetSource sheet="%s" ref="A1:%s%d"/>
                  </cacheSource>
@@ -775,7 +794,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
                  </cacheFields>
                </pivotCacheDefinition>
                """,
-            StreamedPoiSheetWriter.cleanseValue(labelViewsByName.get(dataView.getName())),
+            StreamedSheetWriter.cleanseValue(labelViewsByName.get(dataView.getName())),
             CellReference.convertNumToColString(dataView.getColumns().size() - 1),
             rowsPerView.get(dataView.getName()),
             dataView.getColumns().size(),

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedSheetWriter.java

@@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
 
 import java.io.IOException;
 import java.io.Writer;
+import java.util.HashMap;
+import java.util.Map;
 import org.apache.poi.ss.util.CellReference;
 
 
@@ -31,7 +33,7 @@ import org.apache.poi.ss.util.CellReference;
  ** Write excel formatted XML to a Writer.
  ** Originally from https://coderanch.com/t/548897/java/Generate-large-excel-POI
  *******************************************************************************/
-public class StreamedPoiSheetWriter
+public class StreamedSheetWriter
 {
    private final Writer writer;
    private       int    rowNo;
@@ -41,7 +43,7 @@ public class StreamedPoiSheetWriter
    /*******************************************************************************
     **
     *******************************************************************************/
-   public StreamedPoiSheetWriter(Writer writer)
+   public StreamedSheetWriter(Writer writer)
    {
       this.writer = writer;
    }
@@ -125,14 +127,44 @@ public class StreamedPoiSheetWriter
    {
       if(value != null)
       {
-         if(value.indexOf('&') > -1 || value.indexOf('<') > -1 || value.indexOf('>') > -1 || value.indexOf('\'') > -1 || value.indexOf('"') > -1)
+         StringBuilder rs = new StringBuilder();
+         for(int i = 0; i < value.length(); i++)
          {
-            value = value.replace("&", "&amp;");
-            value = value.replace("<", "&lt;");
-            value = value.replace(">", "&gt;");
-            value = value.replace("'", "&apos;");
-            value = value.replace("\"", "&quot;");
+            char c = value.charAt(i);
+            if(c == '&')
+            {
+               rs.append("&amp;");
+            }
+            else if(c == '<')
+            {
+               rs.append("&lt;");
+            }
+            else if(c == '>')
+            {
+               rs.append("&gt;");
+            }
+            else if(c == '\'')
+            {
+               rs.append("&apos;");
+            }
+            else if(c == '"')
+            {
+               rs.append("&quot;");
+            }
+            else if (c < 32 && c != '\t' && c != '\n')
+            {
+               rs.append(' ');
+            }
+            else
+            {
+               rs.append(c);
+            }
          }
+
+         Map<String, Integer> m = new HashMap();
+         m.computeIfAbsent("s", (s) -> 3);
+
+         value = rs.toString();
       }
 
       return (value);
@@ -207,8 +239,4 @@ public class StreamedPoiSheetWriter
       writer.write("</c>");
    }
 
-   // todo? public void createCell(int columnIndex, Calendar value, int styleIndex) throws IOException
-   // todo? {
-   // todo?    createCell(columnIndex, DateUtil.getExcelDate(value, false), styleIndex);
-   // todo? }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java

@@ -470,7 +470,7 @@ public class DeleteAction
                QRecord recordWithError = new QRecord();
                recordsWithErrors.add(recordWithError);
                recordWithError.setValue(primaryKeyField.getName(), primaryKeyValue);
-               recordWithError.addError(new NotFoundStatusMessage("No record was found to delete for " + primaryKeyField.getLabel() + " = " + primaryKeyValue));
+               recordWithError.addError(new NotFoundStatusMessage("No record was found to delete for " + Objects.requireNonNullElse(primaryKeyField.getLabel(), primaryKeyField.getName()) + " = " + primaryKeyValue));
             }
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -62,6 +62,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.DuplicateKeyBadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@@ -414,7 +415,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
                Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
                if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisList.get(uniqueKey).contains(keyValues.get())))
                {
-                  record.addError(new BadInputStatusMessage("Another record already exists with this " + uniqueKey.getDescription(table)));
+                  record.addError(new DuplicateKeyBadInputStatusMessage("Another record already exists with this " + uniqueKey.getDescription(table)));
                   foundDupe = true;
                   break;
                }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java

@@ -47,6 +47,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import org.apache.commons.lang.BooleanUtils;
 
 
 /*******************************************************************************
@@ -79,9 +80,11 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
 
       try
       {
-         QTableMetaData table           = input.getTable();
-         UniqueKey      uniqueKey       = input.getKey();
-         String         primaryKeyField = table.getPrimaryKeyField();
+         QTableMetaData table                     = input.getTable();
+         UniqueKey      uniqueKey                 = input.getKey();
+         String         primaryKeyField           = table.getPrimaryKeyField();
+         boolean        allowNullKeyValuesToEqual = BooleanUtils.isTrue(input.getAllowNullKeyValuesToEqual());
+
          if(transaction == null)
          {
             transaction = QBackendTransaction.openFor(new InsertInput(input.getTableName()));
@@ -98,10 +101,11 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
             // originally it was thought that we'd need to pass the filter in here           //
             // but, it's been decided not to.  the filter only applies to what we can delete //
             ///////////////////////////////////////////////////////////////////////////////////
-            Map<List<Serializable>, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey);
+            Map<List<Serializable>, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey, allowNullKeyValuesToEqual);
+
             for(QRecord record : page)
             {
-               Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
+               Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
                if(keyValues.isPresent())
                {
                   if(existingKeys.containsKey(keyValues.get()))

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java

@@ -79,7 +79,7 @@ public class StorageAction
 
       if(storageInput.getTableName() == null)
       {
-         throw (new QException("Table name was not specified in query input"));
+         throw (new QException("Table name was not specified in storage input"));
       }
 
       QTableMetaData table = storageInput.getTable();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -68,6 +68,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.model.statusmessages.NotFoundStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
 import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@@ -393,7 +394,12 @@ public class UpdateAction
                   QRecord      oldRecord = lookedUpRecords.get(value);
                   QFieldType   fieldType = table.getField(lock.getFieldName()).getType();
                   Serializable lockValue = ValueUtils.getValueAsFieldType(fieldType, oldRecord.getValue(lock.getFieldName()));
-                  ValidateRecordSecurityLockHelper.validateRecordSecurityValue(table, record, lock, lockValue, fieldType, ValidateRecordSecurityLockHelper.Action.UPDATE);
+
+                  List<QErrorMessage> errors = ValidateRecordSecurityLockHelper.validateRecordSecurityValue(table, lock, lockValue, fieldType, ValidateRecordSecurityLockHelper.Action.UPDATE);
+                  if(CollectionUtils.nullSafeHasContents(errors))
+                  {
+                     errors.forEach(e -> record.addError(e));
+                  }
                }
             }
          }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java

@@ -54,7 +54,7 @@ public class UniqueKeyHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static Map<List<Serializable>, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List<QRecord> recordList, UniqueKey uniqueKey) throws QException
+   public static Map<List<Serializable>, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List<QRecord> recordList, UniqueKey uniqueKey, boolean allowNullKeyValuesToEqual) throws QException
    {
       List<String>                          ukFieldNames    = uniqueKey.getFieldNames();
       Map<List<Serializable>, Serializable> existingRecords = new HashMap<>();
@@ -112,7 +112,7 @@ public class UniqueKeyHelper
          QueryOutput queryOutput = new QueryAction().execute(queryInput);
          for(QRecord record : queryOutput.getRecords())
          {
-            Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record);
+            Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
             if(keyValues.isPresent())
             {
                existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField()));
@@ -128,7 +128,17 @@ public class UniqueKeyHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static Optional<List<Serializable>> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record)
+   public static Map<List<Serializable>, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List<QRecord> recordList, UniqueKey uniqueKey) throws QException
+   {
+      return (getExistingKeys(transaction, table, recordList, uniqueKey, false));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Optional<List<Serializable>> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record, boolean allowNullKeyValuesToEqual)
    {
       try
       {
@@ -138,7 +148,19 @@ public class UniqueKeyHelper
             QFieldMetaData field      = table.getField(fieldName);
             Serializable   value      = record.getValue(fieldName);
             Serializable   typedValue = ValueUtils.getValueAsFieldType(field.getType(), value);
-            keyValues.add(typedValue == null ? new NullUniqueKeyValue() : typedValue);
+
+            ///////////////////////////////////////////////////////////////////////////////////
+            // if null value, look at flag to determine if a null should be used (which will //
+            // allow keys to match), or a NullUniqueKeyValue, (which will never match)       //
+            ///////////////////////////////////////////////////////////////////////////////////
+            if(typedValue == null)
+            {
+               keyValues.add(allowNullKeyValuesToEqual ? null : new NullUniqueKeyValue());
+            }
+            else
+            {
+               keyValues.add(typedValue);
+            }
          }
          return (Optional.of(keyValues));
       }
@@ -150,6 +172,16 @@ public class UniqueKeyHelper
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Optional<List<Serializable>> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record)
+   {
+      return (getKeyValues(table, uniqueKey, record, false));
+   }
+
+
+
    /*******************************************************************************
     ** To make a list of unique key values here behave like they do in an RDBMS
     ** (which is what we're trying to mimic - which is - 2 null values in a field

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.tables.helpers;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -42,13 +43,16 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.security.NullValueBehaviorUtil;
 import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.statusmessages.PermissionDeniedMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@@ -81,160 +85,273 @@ public class ValidateRecordSecurityLockHelper
     *******************************************************************************/
    public static void validateSecurityFields(QTableMetaData table, List<QRecord> records, Action action) throws QException
    {
-      List<RecordSecurityLock> locksToCheck = getRecordSecurityLocks(table, action);
-      if(CollectionUtils.nullSafeIsEmpty(locksToCheck))
+      MultiRecordSecurityLock locksToCheck = getRecordSecurityLocks(table, action);
+      if(locksToCheck == null || CollectionUtils.nullSafeIsEmpty(locksToCheck.getLocks()))
+      {
+         return;
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // we will be relying on primary keys being set in records - but (at least for inserts) //
+      // we might not have pkeys - so make them up (and clear them out at the end)            //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      Map<Serializable, QRecord> madeUpPrimaryKeys = makeUpPrimaryKeysIfNeeded(records, table);
+
+      ////////////////////////////////
+      // actually check lock values //
+      ////////////////////////////////
+      Map<Serializable, RecordWithErrors> errorRecords = new HashMap<>();
+      evaluateRecordLocks(table, records, action, locksToCheck, errorRecords, new ArrayList<>());
+
+      /////////////////////////////////
+      // propagate errors to records //
+      /////////////////////////////////
+      for(RecordWithErrors recordWithErrors : errorRecords.values())
+      {
+         recordWithErrors.propagateErrorsToRecord(locksToCheck);
+      }
+
+      /////////////////////////////////
+      // remove made-up primary keys //
+      /////////////////////////////////
+      String primaryKeyField = table.getPrimaryKeyField();
+      for(QRecord record : madeUpPrimaryKeys.values())
+      {
+         record.setValue(primaryKeyField, null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** For a list of `records` from a `table`, and a given `action`, evaluate a
+    ** `recordSecurityLock` (which may be a multi-lock) - populating the input map
+    ** of `errorRecords` - key'ed by primary key value (real or made up), with
+    ** error messages existing in a tree, with positions matching the multi-lock
+    ** tree that we're navigating, as tracked by `treePosition`.
+    **
+    ** Recursively processes multi-locks (and the top-level call is always with a
+    ** multi-lock - as the table's recordLocks list converted to an AND-multi-lock).
+    **
+    ** Of note - for the case of READ_WRITE locks, we're only evaluating the values
+    ** on the record, to see if they're allowed for us to store (because if we didn't
+    ** have the key, we wouldn't have been able to read the value (which is verified
+    ** outside of here, in UpdateAction/DeleteAction).
+    **
+    ** BUT - WRITE locks - in their case, we read the record no matter what, and in
+    ** here we need to verify we have a key that allows us to WRITE the record.
+    *******************************************************************************/
+   private static void evaluateRecordLocks(QTableMetaData table, List<QRecord> records, Action action, RecordSecurityLock recordSecurityLock, Map<Serializable, RecordWithErrors> errorRecords, List<Integer> treePosition) throws QException
+   {
+      if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock)
+      {
+         /////////////////////////////////////////////
+         // for multi-locks, make recursive descent //
+         /////////////////////////////////////////////
+         int i = 0;
+         for(RecordSecurityLock childLock : CollectionUtils.nonNullList(multiRecordSecurityLock.getLocks()))
+         {
+            treePosition.add(i);
+            evaluateRecordLocks(table, records, action, childLock, errorRecords, treePosition);
+            treePosition.remove(treePosition.size() - 1);
+            i++;
+         }
+
+         return;
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if this lock has an all-access key, and the user has that key, then there can't be any errors here, so return early //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      QSecurityKeyType securityKeyType = QContext.getQInstance().getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
+      if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()) && QContext.getQSession().hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
       {
          return;
       }
 
       ////////////////////////////////
-      // actually check lock values //
+      // proceed w/ non-multi locks //
       ////////////////////////////////
-      for(RecordSecurityLock recordSecurityLock : locksToCheck)
+      String primaryKeyField = table.getPrimaryKeyField();
+      if(CollectionUtils.nullSafeIsEmpty(recordSecurityLock.getJoinNameChain()))
       {
-         if(CollectionUtils.nullSafeIsEmpty(recordSecurityLock.getJoinNameChain()))
+         //////////////////////////////////////////////////////////////////////////////////
+         // handle the value being in the table we're inserting/updating (e.g., no join) //
+         //////////////////////////////////////////////////////////////////////////////////
+         QFieldMetaData field = table.getField(recordSecurityLock.getFieldName());
+
+         for(QRecord record : records)
          {
-            //////////////////////////////////////////////////////////////////////////////////
-            // handle the value being in the table we're inserting/updating (e.g., no join) //
-            //////////////////////////////////////////////////////////////////////////////////
-            QFieldMetaData field = table.getField(recordSecurityLock.getFieldName());
-
-            for(QRecord record : records)
+            if(action.equals(Action.UPDATE) && !record.getValues().containsKey(field.getName()) && RecordSecurityLock.LockScope.READ_AND_WRITE.equals(recordSecurityLock.getLockScope()))
             {
-               if(action.equals(Action.UPDATE) && !record.getValues().containsKey(field.getName()) && RecordSecurityLock.LockScope.READ_AND_WRITE.equals(recordSecurityLock.getLockScope()))
-               {
-                  /////////////////////////////////////////////////////////////////////////////////////////////////////////
-                  // if this is a read-write lock, then if we have the record, it means we were able to read the record. //
-                  // So if we're not updating the security field, then no error can come from it!                        //
-                  /////////////////////////////////////////////////////////////////////////////////////////////////////////
-                  continue;
-               }
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // if this is a read-write lock, then if we have the record, it means we were able to read the record. //
+               // So if we're not updating the security field, then no error can come from it!                        //
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////
+               continue;
+            }
 
-               Serializable recordSecurityValue = record.getValue(field.getName());
-               validateRecordSecurityValue(table, record, recordSecurityLock, recordSecurityValue, field.getType(), action);
+            Serializable        recordSecurityValue = record.getValue(field.getName());
+            List<QErrorMessage> recordErrors        = validateRecordSecurityValue(table, recordSecurityLock, recordSecurityValue, field.getType(), action);
+            if(CollectionUtils.nullSafeHasContents(recordErrors))
+            {
+               errorRecords.computeIfAbsent(record.getValue(primaryKeyField), (k) -> new RecordWithErrors(record)).addAll(recordErrors, treePosition);
             }
          }
-         else
+      }
+      else
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // else look for the joined record - if it isn't found, assume a fail - else validate security value if found //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         QJoinMetaData leftMostJoin  = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(0));
+         QJoinMetaData rightMostJoin = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(recordSecurityLock.getJoinNameChain().size() - 1));
+
+         ////////////////////////////////
+         // todo probably, but more... //
+         ////////////////////////////////
+         // if(leftMostJoin.getLeftTable().equals(table.getName()))
+         // {
+         //    leftMostJoin = leftMostJoin.flip();
+         // }
+
+         QTableMetaData rightMostJoinTable = QContext.getQInstance().getTable(rightMostJoin.getRightTable());
+         QTableMetaData leftMostJoinTable  = QContext.getQInstance().getTable(leftMostJoin.getLeftTable());
+
+         for(List<QRecord> inputRecordPage : CollectionUtils.getPages(records, 500))
          {
-            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            // else look for the joined record - if it isn't found, assume a fail - else validate security value if found //
-            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            QJoinMetaData  leftMostJoin       = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(0));
-            QJoinMetaData  rightMostJoin      = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(recordSecurityLock.getJoinNameChain().size() - 1));
-            QTableMetaData rightMostJoinTable = QContext.getQInstance().getTable(rightMostJoin.getRightTable());
-            QTableMetaData leftMostJoinTable  = QContext.getQInstance().getTable(leftMostJoin.getLeftTable());
+            ////////////////////////////////////////////////////////////////////////////////////////////////
+            // set up a query for joined records                                                          //
+            // query will be like (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) //
+            ////////////////////////////////////////////////////////////////////////////////////////////////
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(leftMostJoin.getLeftTable());
+            QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
+            queryInput.setFilter(filter);
 
-            for(List<QRecord> inputRecordPage : CollectionUtils.getPages(records, 500))
+            for(String joinName : recordSecurityLock.getJoinNameChain())
             {
-               ////////////////////////////////////////////////////////////////////////////////////////////////
-               // set up a query for joined records                                                          //
-               // query will be like (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) //
-               ////////////////////////////////////////////////////////////////////////////////////////////////
-               QueryInput queryInput = new QueryInput();
-               queryInput.setTableName(leftMostJoin.getLeftTable());
-               QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
-               queryInput.setFilter(filter);
-
-               for(String joinName : recordSecurityLock.getJoinNameChain())
+               ///////////////////////////////////////
+               // we don't need the right-most join //
+               ///////////////////////////////////////
+               if(!joinName.equals(rightMostJoin.getName()))
                {
-                  ///////////////////////////////////////
-                  // we don't need the right-most join //
-                  ///////////////////////////////////////
-                  if(!joinName.equals(rightMostJoin.getName()))
+                  queryInput.withQueryJoin(new QueryJoin().withJoinMetaData(QContext.getQInstance().getJoin(joinName)).withSelect(true));
+               }
+            }
+
+            ///////////////////////////////////////////////////////////////////////////////////////////////////
+            // foreach input record (in this page), put it in a listing hash, with key = list of join-values //
+            // e.g., (17,47)=(QRecord1), (18,48)=(QRecord2,QRecord3)                                         //
+            // also build up the query's sub-filters here (only adding them if they're unique).              //
+            // e.g., 2 order-lines referencing the same orderId don't need to be added to the query twice    //
+            ///////////////////////////////////////////////////////////////////////////////////////////////////
+            ListingHash<List<Serializable>, QRecord> inputRecordMapByJoinFields = new ListingHash<>();
+            for(QRecord inputRecord : inputRecordPage)
+            {
+               List<Serializable> inputRecordJoinValues = new ArrayList<>();
+               QQueryFilter       subFilter             = new QQueryFilter();
+
+               boolean updatingAnyLockJoinFields = false;
+               for(JoinOn joinOn : rightMostJoin.getJoinOns())
+               {
+                  QFieldType   type             = rightMostJoinTable.getField(joinOn.getRightField()).getType();
+                  Serializable inputRecordValue = ValueUtils.getValueAsFieldType(type, inputRecord.getValue(joinOn.getRightField()));
+                  inputRecordJoinValues.add(inputRecordValue);
+
+                  // if we have a value in this field (and it's not the primary key), then it means we're updating part of the lock
+                  if(inputRecordValue != null && !joinOn.getRightField().equals(table.getPrimaryKeyField()))
                   {
-                     queryInput.withQueryJoin(new QueryJoin().withJoinMetaData(QContext.getQInstance().getJoin(joinName)).withSelect(true));
+                     updatingAnyLockJoinFields = true;
                   }
+
+                  subFilter.addCriteria(inputRecordValue == null
+                     ? new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.IS_BLANK)
+                     : new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.EQUALS, inputRecordValue));
                }
 
-               ///////////////////////////////////////////////////////////////////////////////////////////////////
-               // foreach input record (in this page), put it in a listing hash, with key = list of join-values //
-               // e.g., (17,47)=(QRecord1), (18,48)=(QRecord2,QRecord3)                                         //
-               // also build up the query's sub-filters here (only adding them if they're unique).              //
-               // e.g., 2 order-lines referencing the same orderId don't need to be added to the query twice    //
-               ///////////////////////////////////////////////////////////////////////////////////////////////////
-               ListingHash<List<Serializable>, QRecord> inputRecordMapByJoinFields = new ListingHash<>();
-               for(QRecord inputRecord : inputRecordPage)
+               //////////////////////////////////
+               // todo maybe, some version of? //
+               //////////////////////////////////
+               // if(action.equals(Action.UPDATE) && !updatingAnyLockJoinFields && RecordSecurityLock.LockScope.READ_AND_WRITE.equals(recordSecurityLock.getLockScope()))
+               // {
+               //    /////////////////////////////////////////////////////////////////////////////////////////////////////////
+               //    // if this is a read-write lock, then if we have the record, it means we were able to read the record. //
+               //    // So if we're not updating the security field, then no error can come from it!                        //
+               //    /////////////////////////////////////////////////////////////////////////////////////////////////////////
+               //    continue;
+               // }
+
+               if(!inputRecordMapByJoinFields.containsKey(inputRecordJoinValues))
                {
-                  List<Serializable> inputRecordJoinValues = new ArrayList<>();
-                  QQueryFilter       subFilter             = new QQueryFilter();
-
-                  for(JoinOn joinOn : rightMostJoin.getJoinOns())
-                  {
-                     QFieldType   type             = rightMostJoinTable.getField(joinOn.getRightField()).getType();
-                     Serializable inputRecordValue = ValueUtils.getValueAsFieldType(type, inputRecord.getValue(joinOn.getRightField()));
-                     inputRecordJoinValues.add(inputRecordValue);
-
-                     subFilter.addCriteria(inputRecordValue == null
-                        ? new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.IS_BLANK)
-                        : new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.EQUALS, inputRecordValue));
-                  }
-
-                  if(!inputRecordMapByJoinFields.containsKey(inputRecordJoinValues))
-                  {
-                     ////////////////////////////////////////////////////////////////////////////////
-                     // only add this sub-filter if it's for a list of keys we haven't seen before //
-                     ////////////////////////////////////////////////////////////////////////////////
-                     filter.addSubFilter(subFilter);
-                  }
-
-                  inputRecordMapByJoinFields.add(inputRecordJoinValues, inputRecord);
+                  ////////////////////////////////////////////////////////////////////////////////
+                  // only add this sub-filter if it's for a list of keys we haven't seen before //
+                  ////////////////////////////////////////////////////////////////////////////////
+                  filter.addSubFilter(subFilter);
                }
 
-               //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               // execute the query for joined records - then put them in a map with keys corresponding to the join values //
-               // e.g., (17,47)=(JoinRecord), (18,48)=(JoinRecord)                                                         //
-               //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               QueryOutput                      queryOutput               = new QueryAction().execute(queryInput);
-               Map<List<Serializable>, QRecord> joinRecordMapByJoinFields = new HashMap<>();
-               for(QRecord joinRecord : queryOutput.getRecords())
-               {
-                  List<Serializable> joinRecordValues = new ArrayList<>();
-                  for(JoinOn joinOn : rightMostJoin.getJoinOns())
-                  {
-                     Serializable joinValue = joinRecord.getValue(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField());
-                     if(joinValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> !n.contains(".")))
-                     {
-                        joinValue = joinRecord.getValue(joinOn.getLeftField());
-                     }
-                     joinRecordValues.add(joinValue);
-                  }
+               inputRecordMapByJoinFields.add(inputRecordJoinValues, inputRecord);
+            }
 
-                  joinRecordMapByJoinFields.put(joinRecordValues, joinRecord);
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // execute the query for joined records - then put them in a map with keys corresponding to the join values //
+            // e.g., (17,47)=(JoinRecord), (18,48)=(JoinRecord)                                                         //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            QueryOutput                      queryOutput               = new QueryAction().execute(queryInput);
+            Map<List<Serializable>, QRecord> joinRecordMapByJoinFields = new HashMap<>();
+            for(QRecord joinRecord : queryOutput.getRecords())
+            {
+               List<Serializable> joinRecordValues = new ArrayList<>();
+               for(JoinOn joinOn : rightMostJoin.getJoinOns())
+               {
+                  Serializable joinValue = joinRecord.getValue(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField());
+                  if(joinValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> !n.contains(".")))
+                  {
+                     joinValue = joinRecord.getValue(joinOn.getLeftField());
+                  }
+                  joinRecordValues.add(joinValue);
                }
 
-               //////////////////////////////////////////////////////////////////////////////////////////////////
-               // now for each input record, look for its joinRecord - if it isn't found, then this insert     //
-               // isn't allowed.  if it is found, then validate its value matches this session's security keys //
-               //////////////////////////////////////////////////////////////////////////////////////////////////
-               for(Map.Entry<List<Serializable>, List<QRecord>> entry : inputRecordMapByJoinFields.entrySet())
+               joinRecordMapByJoinFields.put(joinRecordValues, joinRecord);
+            }
+
+            //////////////////////////////////////////////////////////////////////////////////////////////////
+            // now for each input record, look for its joinRecord - if it isn't found, then this insert     //
+            // isn't allowed.  if it is found, then validate its value matches this session's security keys //
+            //////////////////////////////////////////////////////////////////////////////////////////////////
+            for(Map.Entry<List<Serializable>, List<QRecord>> entry : inputRecordMapByJoinFields.entrySet())
+            {
+               List<Serializable> inputRecordJoinValues = entry.getKey();
+               List<QRecord>      inputRecords          = entry.getValue();
+               if(joinRecordMapByJoinFields.containsKey(inputRecordJoinValues))
                {
-                  List<Serializable> inputRecordJoinValues = entry.getKey();
-                  List<QRecord>      inputRecords          = entry.getValue();
-                  if(joinRecordMapByJoinFields.containsKey(inputRecordJoinValues))
+                  QRecord joinRecord = joinRecordMapByJoinFields.get(inputRecordJoinValues);
+
+                  String         fieldName           = recordSecurityLock.getFieldName().replaceFirst(".*\\.", "");
+                  QFieldMetaData field               = leftMostJoinTable.getField(fieldName);
+                  Serializable   recordSecurityValue = joinRecord.getValue(fieldName);
+                  if(recordSecurityValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> n.contains(".")))
                   {
-                     QRecord joinRecord = joinRecordMapByJoinFields.get(inputRecordJoinValues);
+                     recordSecurityValue = joinRecord.getValue(recordSecurityLock.getFieldName());
+                  }
 
-                     String         fieldName           = recordSecurityLock.getFieldName().replaceFirst(".*\\.", "");
-                     QFieldMetaData field               = leftMostJoinTable.getField(fieldName);
-                     Serializable   recordSecurityValue = joinRecord.getValue(fieldName);
-                     if(recordSecurityValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> n.contains(".")))
+                  for(QRecord inputRecord : inputRecords)
+                  {
+                     List<QErrorMessage> recordErrors = validateRecordSecurityValue(table, recordSecurityLock, recordSecurityValue, field.getType(), action);
+                     if(CollectionUtils.nullSafeHasContents(recordErrors))
                      {
-                        recordSecurityValue = joinRecord.getValue(recordSecurityLock.getFieldName());
-                     }
-
-                     for(QRecord inputRecord : inputRecords)
-                     {
-                        validateRecordSecurityValue(table, inputRecord, recordSecurityLock, recordSecurityValue, field.getType(), action);
+                        errorRecords.computeIfAbsent(inputRecord.getValue(primaryKeyField), (k) -> new RecordWithErrors(inputRecord)).addAll(recordErrors, treePosition);
                      }
                   }
-                  else
+               }
+               else
+               {
+                  for(QRecord inputRecord : inputRecords)
                   {
-                     for(QRecord inputRecord : inputRecords)
+                     if(RecordSecurityLock.NullValueBehavior.DENY.equals(NullValueBehaviorUtil.getEffectiveNullValueBehavior(recordSecurityLock)))
                      {
-                        if(RecordSecurityLock.NullValueBehavior.DENY.equals(NullValueBehaviorUtil.getEffectiveNullValueBehavior(recordSecurityLock)))
-                        {
-                           inputRecord.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record - the referenced " + leftMostJoinTable.getLabel() + " was not found."));
-                        }
+                        PermissionDeniedMessage error = new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record - the referenced " + leftMostJoinTable.getLabel() + " was not found.");
+                        errorRecords.computeIfAbsent(inputRecord.getValue(primaryKeyField), (k) -> new RecordWithErrors(inputRecord)).add(error, treePosition);
                      }
                   }
                }
@@ -246,45 +363,81 @@ public class ValidateRecordSecurityLockHelper
 
 
    /*******************************************************************************
-    **
+    ** for tracking errors, we use primary keys.  add "made up" ones to records
+    ** if needed (e.g., insert use-case).
     *******************************************************************************/
-   private static List<RecordSecurityLock> getRecordSecurityLocks(QTableMetaData table, Action action)
+   private static Map<Serializable, QRecord> makeUpPrimaryKeysIfNeeded(List<QRecord> records, QTableMetaData table)
    {
-      List<RecordSecurityLock> recordSecurityLocks = CollectionUtils.nonNullList(table.getRecordSecurityLocks());
-      List<RecordSecurityLock> locksToCheck        = new ArrayList<>();
-
-      recordSecurityLocks = switch(action)
+      String                     primaryKeyField   = table.getPrimaryKeyField();
+      Map<Serializable, QRecord> madeUpPrimaryKeys = new HashMap<>();
+      Integer                    madeUpPrimaryKey  = -1;
+      for(QRecord record : records)
       {
-         case INSERT, UPDATE, DELETE -> RecordSecurityLockFilters.filterForWriteLocks(recordSecurityLocks);
-         case SELECT -> RecordSecurityLockFilters.filterForReadLocks(recordSecurityLocks);
+         if(record.getValue(primaryKeyField) == null)
+         {
+            madeUpPrimaryKeys.put(madeUpPrimaryKey, record);
+            record.setValue(primaryKeyField, madeUpPrimaryKey);
+            madeUpPrimaryKey--;
+         }
+      }
+      return madeUpPrimaryKeys;
+   }
+
+
+
+   /*******************************************************************************
+    ** For a given table & action type, convert the table's record locks to a
+    ** MultiRecordSecurityLock, with only the appropriate lock-scopes being included
+    ** (e.g., read-locks for selects, write-locks for insert/update/delete).
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:Indentation")
+   static MultiRecordSecurityLock getRecordSecurityLocks(QTableMetaData table, Action action)
+   {
+      List<RecordSecurityLock> allLocksOnTable = CollectionUtils.nonNullList(table.getRecordSecurityLocks());
+      MultiRecordSecurityLock locksOfType = switch(action)
+      {
+         case INSERT, UPDATE, DELETE -> RecordSecurityLockFilters.filterForWriteLockTree(allLocksOnTable);
+         case SELECT -> RecordSecurityLockFilters.filterForReadLockTree(allLocksOnTable);
          default -> throw (new IllegalArgumentException("Unsupported action: " + action));
       };
 
+      if(action.equals(Action.UPDATE))
+      {
+         ////////////////////////////////////////////////////////////////////////////
+         // todo at some point this seemed right, but now it doesn't - needs work. //
+         ////////////////////////////////////////////////////////////////////////////
+         // ////////////////////////////////////////////////////////
+         // // when doing an update, convert all OR's to AND's... //
+         // ////////////////////////////////////////////////////////
+         // updateOperators(locksOfType, MultiRecordSecurityLock.BooleanOperator.AND);
+      }
+
       ////////////////////////////////////////
       // if there are no locks, just return //
       ////////////////////////////////////////
-      if(CollectionUtils.nullSafeIsEmpty(recordSecurityLocks))
+      if(locksOfType == null || CollectionUtils.nullSafeIsEmpty(locksOfType.getLocks()))
       {
          return (null);
       }
 
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // decide if any locks need checked - where one may not need checked if it has an all-access key, and the user has all-access //
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      for(RecordSecurityLock recordSecurityLock : recordSecurityLocks)
+      return (locksOfType);
+   }
+
+
+
+   /*******************************************************************************
+    ** for a full multi-lock tree, set all of the boolean operators to the specified one.
+    *******************************************************************************/
+   private static void updateOperators(MultiRecordSecurityLock multiLock, MultiRecordSecurityLock.BooleanOperator operator)
+   {
+      multiLock.setOperator(operator);
+      for(RecordSecurityLock childLock : multiLock.getLocks())
       {
-         QSecurityKeyType securityKeyType = QContext.getQInstance().getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
-         if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()) && QContext.getQSession().hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
+         if(childLock instanceof MultiRecordSecurityLock childMultiLock)
          {
-            LOG.trace("Session has " + securityKeyType.getAllAccessKeyName() + " - not checking this lock.");
-         }
-         else
-         {
-            locksToCheck.add(recordSecurityLock);
+            updateOperators(childMultiLock, operator);
          }
       }
-
-      return (locksToCheck);
    }
 
 
@@ -292,7 +445,7 @@ public class ValidateRecordSecurityLockHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static void validateRecordSecurityValue(QTableMetaData table, QRecord record, RecordSecurityLock recordSecurityLock, Serializable recordSecurityValue, QFieldType fieldType, Action action)
+   public static List<QErrorMessage> validateRecordSecurityValue(QTableMetaData table, RecordSecurityLock recordSecurityLock, Serializable recordSecurityValue, QFieldType fieldType, Action action)
    {
       if(recordSecurityValue == null)
       {
@@ -302,7 +455,7 @@ public class ValidateRecordSecurityLockHelper
          if(RecordSecurityLock.NullValueBehavior.DENY.equals(NullValueBehaviorUtil.getEffectiveNullValueBehavior(recordSecurityLock)))
          {
             String lockLabel = CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain()) ? recordSecurityLock.getSecurityKeyType() : table.getField(recordSecurityLock.getFieldName()).getLabel();
-            record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record without a value in the field: " + lockLabel));
+            return (List.of(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record without a value in the field: " + lockLabel)));
          }
       }
       else
@@ -314,15 +467,305 @@ public class ValidateRecordSecurityLockHelper
                ///////////////////////////////////////////////////////////////////////////////////////////////
                // avoid telling the user a value from a foreign record that they didn't pass in themselves. //
                ///////////////////////////////////////////////////////////////////////////////////////////////
-               record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record."));
+               return (List.of(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record.")));
             }
             else
             {
                QFieldMetaData field = table.getField(recordSecurityLock.getFieldName());
-               record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record with a value of " + recordSecurityValue + " in the field: " + field.getLabel()));
+               return (List.of(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record with a value of " + recordSecurityValue + " in the field: " + field.getLabel())));
             }
          }
       }
+      return (Collections.emptyList());
+   }
+
+
+
+   /*******************************************************************************
+    ** Class to track errors that we're associating with a record.
+    **
+    ** More complex than it first seems to be needed, because as we're evaluating
+    ** locks, we might find some, but based on the boolean condition associated with
+    ** them, they might not actually be record-level errors.
+    **
+    ** e.g., two locks with an OR relationship - as long as one passes, the record
+    ** should have no errors.  And so-on through the tree of locks/multi-locks.
+    **
+    ** Stores the errors in a tree of ErrorTreeNode objects.
+    **
+    ** References into that tree are achieved via a List of Integer called "tree positions"
+    ** where each entry in the list denotes the index of the tree node at that level.
+    **
+    ** e.g., given this tree:
+    ** <pre>
+    **   A      B
+    **  / \    /|\
+    ** C   D  E F G
+    **     |
+    **     H
+    ** </pre>
+    **
+    ** The positions of each node would be:
+    ** <pre>
+    ** A: [0]
+    ** B: [1]
+    ** C: [0,0]
+    ** D: [0,1]
+    ** E: [1,0]
+    ** F: [1,1]
+    ** G: [1,2]
+    ** H: [0,1,0]
+    ** </pre>
+    *******************************************************************************/
+   static class RecordWithErrors
+   {
+      private QRecord       record;
+      private ErrorTreeNode errorTree;
+
+
+
+      /*******************************************************************************
+       ** Constructor
+       **
+       *******************************************************************************/
+      public RecordWithErrors(QRecord record)
+      {
+         this.record = record;
+      }
+
+
+
+      /*******************************************************************************
+       ** add a list of errors, for a given list of tree positions
+       *******************************************************************************/
+      public void addAll(List<QErrorMessage> recordErrors, List<Integer> treePositions)
+      {
+         if(errorTree == null)
+         {
+            errorTree = new ErrorTreeNode();
+         }
+
+         ErrorTreeNode node = errorTree;
+         for(Integer treePosition : treePositions)
+         {
+            if(node.children == null)
+            {
+               node.children = new ArrayList<>(treePosition);
+            }
+
+            while(treePosition >= node.children.size())
+            {
+               node.children.add(null);
+            }
+
+            if(node.children.get(treePosition) == null)
+            {
+               node.children.set(treePosition, new ErrorTreeNode());
+            }
+
+            node = node.children.get(treePosition);
+         }
+
+         if(node.errors == null)
+         {
+            node.errors = new ArrayList<>();
+         }
+         node.errors.addAll(recordErrors);
+      }
+
+
+
+      /*******************************************************************************
+       ** add a single error to a given tree-position
+       *******************************************************************************/
+      public void add(QErrorMessage error, List<Integer> treePositions)
+      {
+         addAll(List.of(error), treePositions);
+      }
+
+
+
+      /*******************************************************************************
+       ** after the tree of errors has been built - walk a lock-tree (locksToCheck)
+       ** and resolve boolean operations, to get a final list of errors (possibly empty)
+       ** to put on the record.
+       *******************************************************************************/
+      public void propagateErrorsToRecord(MultiRecordSecurityLock locksToCheck)
+      {
+         List<QErrorMessage> errors = recursivePropagation(locksToCheck, new ArrayList<>());
+
+         if(CollectionUtils.nullSafeHasContents(errors))
+         {
+            errors.forEach(e -> record.addError(e));
+         }
+      }
+
+
+
+      /*******************************************************************************
+       ** recursive implementation of the propagation method - e.g., walk tree applying
+       ** boolean logic.
+       *******************************************************************************/
+      private List<QErrorMessage> recursivePropagation(MultiRecordSecurityLock locksToCheck, List<Integer> treePositions)
+      {
+         //////////////////////////////////////////////////////////////////
+         // build a list of errors at this level (and deeper levels too) //
+         //////////////////////////////////////////////////////////////////
+         List<QErrorMessage> errorsFromThisLevel = new ArrayList<>();
+
+         int i = 0;
+         for(RecordSecurityLock lock : locksToCheck.getLocks())
+         {
+            List<QErrorMessage> errorsFromThisLock;
+
+            treePositions.add(i);
+            if(lock instanceof MultiRecordSecurityLock childMultiLock)
+            {
+               errorsFromThisLock = recursivePropagation(childMultiLock, treePositions);
+            }
+            else
+            {
+               errorsFromThisLock = getErrorsFromTree(treePositions);
+            }
+
+            errorsFromThisLevel.addAll(errorsFromThisLock);
+
+            treePositions.remove(treePositions.size() - 1);
+            i++;
+         }
+
+         if(MultiRecordSecurityLock.BooleanOperator.AND.equals(locksToCheck.getOperator()))
+         {
+            //////////////////////////////////////////////////////////////
+            // for an AND - if there were ANY errors, then return them. //
+            //////////////////////////////////////////////////////////////
+            if(!errorsFromThisLevel.isEmpty())
+            {
+               return (errorsFromThisLevel);
+            }
+         }
+         else // OR
+         {
+            //////////////////////////////////////////////////////////
+            // for an OR - only return if ALL conditions had errors //
+            //////////////////////////////////////////////////////////
+            if(errorsFromThisLevel.size() == locksToCheck.getLocks().size())
+            {
+               return (errorsFromThisLevel); // todo something smarter?
+            }
+         }
+
+         ///////////////////////////////////
+         // else - no errors - empty list //
+         ///////////////////////////////////
+         return Collections.emptyList();
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      private List<QErrorMessage> getErrorsFromTree(List<Integer> treePositions)
+      {
+         ErrorTreeNode node = errorTree;
+
+         for(Integer treePosition : treePositions)
+         {
+            if(node.children == null)
+            {
+               return Collections.emptyList();
+            }
+
+            if(treePosition >= node.children.size())
+            {
+               return Collections.emptyList();
+            }
+
+            if(node.children.get(treePosition) == null)
+            {
+               return Collections.emptyList();
+            }
+
+            node = node.children.get(treePosition);
+         }
+
+         if(node.errors == null)
+         {
+            return Collections.emptyList();
+         }
+
+         return node.errors;
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public String toString()
+      {
+         try
+         {
+            return JsonUtils.toPrettyJson(this);
+         }
+         catch(Exception e)
+         {
+            return "error in toString";
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** tree node used by RecordWithErrors
+    *******************************************************************************/
+   static class ErrorTreeNode
+   {
+      private List<QErrorMessage>      errors;
+      private ArrayList<ErrorTreeNode> children;
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public String toString()
+      {
+         try
+         {
+            return JsonUtils.toPrettyJson(this);
+         }
+         catch(Exception e)
+         {
+            return "error in toString";
+         }
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for errors - only here for Jackson/toString
+       **
+       *******************************************************************************/
+      public List<QErrorMessage> getErrors()
+      {
+         return errors;
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for children - only here for Jackson/toString
+       **
+       *******************************************************************************/
+      public ArrayList<ErrorTreeNode> getChildren()
+      {
+         return children;
+      }
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java

@@ -560,20 +560,47 @@ public class QPossibleValueTranslator
     *******************************************************************************/
    private void primePvsCache(String tableName, List<QPossibleValueSource> possibleValueSources, Collection<Serializable> values)
    {
+      String idField = null;
       for(QPossibleValueSource possibleValueSource : possibleValueSources)
       {
          possibleValueCache.putIfAbsent(possibleValueSource.getName(), new HashMap<>());
+         String thisPvsIdField;
+         if(StringUtils.hasContent(possibleValueSource.getOverrideIdField()))
+         {
+            thisPvsIdField = possibleValueSource.getOverrideIdField();
+         }
+         else
+         {
+            thisPvsIdField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField();
+         }
+
+         if(idField == null)
+         {
+            idField = thisPvsIdField;
+         }
+         else
+         {
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // does this ever happen?  maybe not... because, like, the list of values probably wouldn't make sense for //
+            // more than one field in the table...                                                                     //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(!idField.equals(thisPvsIdField))
+            {
+               for(QPossibleValueSource valueSource : possibleValueSources)
+               {
+                  primePvsCache(tableName, List.of(valueSource), values);
+               }
+            }
+         }
       }
 
       try
       {
-         String primaryKeyField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField();
-
          for(List<Serializable> page : CollectionUtils.getPages(values, 1000))
          {
             QueryInput queryInput = new QueryInput();
             queryInput.setTableName(tableName);
-            queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, page)));
+            queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(idField, QCriteriaOperator.IN, page)));
             queryInput.setTransaction(getTransaction(tableName));
 
             /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -618,7 +645,7 @@ public class QPossibleValueTranslator
             ///////////////////////////////////////////////////////////////////////////////////
             for(QRecord record : queryOutput.getRecords())
             {
-               Serializable pkeyValue = record.getValue(primaryKeyField);
+               Serializable pkeyValue = record.getValue(idField);
                for(QPossibleValueSource possibleValueSource : possibleValueSources)
                {
                   QPossibleValue<?> possibleValue = new QPossibleValue<>(pkeyValue, record.getRecordLabel());

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java

@@ -28,7 +28,6 @@ import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -364,7 +363,9 @@ public class QValueFormatter
             }
          }
 
-         setDisplayValuesInRecord(fieldMap, record);
+         ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, QContext.getQInstance(), table, records, null);
+
+         setDisplayValuesInRecord(table, fieldMap, record, true);
          record.setRecordLabel(formatRecordLabel(table, record));
       }
    }
@@ -374,61 +375,49 @@ public class QValueFormatter
    /*******************************************************************************
     ** For a list of records, set their recordLabels and display values
     *******************************************************************************/
-   public static void setDisplayValuesInRecords(Collection<QFieldMetaData> fields, List<QRecord> records)
+   public static void setDisplayValuesInRecords(QTableMetaData table, Map<String, QFieldMetaData> fields, List<QRecord> records)
    {
       if(records == null)
       {
          return;
       }
 
+      if(table != null)
+      {
+         ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, QContext.getQInstance(), table, records, null);
+      }
+
       for(QRecord record : records)
       {
-         setDisplayValuesInRecord(fields, record);
+         setDisplayValuesInRecord(table, fields, record, true);
       }
    }
 
 
 
    /*******************************************************************************
-    ** For a list of records, set their recordLabels and display values
+    ** For a single record, set its display values - public version of this.
     *******************************************************************************/
-   public static void setDisplayValuesInRecords(Map<String, QFieldMetaData> fields, List<QRecord> records)
+   public static void setDisplayValuesInRecord(QTableMetaData table, Map<String, QFieldMetaData> fields, QRecord record)
    {
-      if(records == null)
-      {
-         return;
-      }
-
-      for(QRecord record : records)
-      {
-         setDisplayValuesInRecord(fields, record);
-      }
+      setDisplayValuesInRecord(table, fields, record, false);
    }
 
 
-
    /*******************************************************************************
-    ** For a list of records, set their display values
+    ** For a single record, set its display values - where caller (meant to stay private)
+    ** can specify if they've already done fieldBehaviors (to avoid re-doing).
     *******************************************************************************/
-   public static void setDisplayValuesInRecord(Collection<QFieldMetaData> fields, QRecord record)
+   private static void setDisplayValuesInRecord(QTableMetaData table, Map<String, QFieldMetaData> fields, QRecord record, boolean alreadyAppliedFieldDisplayBehaviors)
    {
-      for(QFieldMetaData field : fields)
+      if(!alreadyAppliedFieldDisplayBehaviors)
       {
-         if(record.getDisplayValue(field.getName()) == null)
+         if(table != null)
          {
-            String formattedValue = formatValue(field, record.getValue(field.getName()));
-            record.setDisplayValue(field.getName(), formattedValue);
+            ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, QContext.getQInstance(), table, List.of(record), null);
          }
       }
-   }
 
-
-
-   /*******************************************************************************
-    ** For a list of records, set their display values
-    *******************************************************************************/
-   public static void setDisplayValuesInRecord(Map<String, QFieldMetaData> fields, QRecord record)
-   {
       for(Map.Entry<String, QFieldMetaData> entry : fields.entrySet())
       {
          String         fieldName = entry.getKey();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java

@@ -275,8 +275,19 @@ public class SearchPossibleValueSourceAction
 
       queryInput.setFilter(queryFilter);
 
-      QueryOutput             queryOutput     = new QueryAction().execute(queryInput);
-      List<Serializable>      ids             = queryOutput.getRecords().stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList();
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+      String fieldName;
+      if(StringUtils.hasContent(possibleValueSource.getOverrideIdField()))
+      {
+         fieldName = possibleValueSource.getOverrideIdField();
+      }
+      else
+      {
+         fieldName = table.getPrimaryKeyField();
+      }
+
+      List<Serializable> ids = queryOutput.getRecords().stream().map(r -> r.getValue(fieldName)).toList();
       List<QPossibleValue<?>> qPossibleValues = possibleValueTranslator.buildTranslatedPossibleValueList(possibleValueSource, ids);
       output.setResults(qPossibleValues);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplier.java

@@ -27,6 +27,7 @@ import java.util.Set;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -44,7 +45,8 @@ public class ValueBehaviorApplier
    public enum Action
    {
       INSERT,
-      UPDATE
+      UPDATE,
+      FORMATTING
    }
 
 
@@ -63,7 +65,34 @@ public class ValueBehaviorApplier
       {
          for(FieldBehavior<?> fieldBehavior : CollectionUtils.nonNullCollection(field.getBehaviors()))
          {
-            fieldBehavior.apply(action, recordList, instance, table, field, behaviorsToOmit);
+            boolean applyBehavior = true;
+            if(behaviorsToOmit != null && behaviorsToOmit.contains(fieldBehavior))
+            {
+               /////////////////////////////////////////////////////////////////////////////////////////
+               // if we're given a set of behaviors to omit, and this behavior is in there, then skip //
+               /////////////////////////////////////////////////////////////////////////////////////////
+               applyBehavior = false;
+            }
+
+            if(Action.FORMATTING == action && !(fieldBehavior instanceof FieldDisplayBehavior<?>))
+            {
+               ////////////////////////////////////////////////////////////////////////////////////////////////
+               // for the formatting action, do not apply the behavior unless it is a field-display-behavior //
+               ////////////////////////////////////////////////////////////////////////////////////////////////
+               applyBehavior = false;
+            }
+            else if(Action.FORMATTING != action && fieldBehavior instanceof FieldDisplayBehavior<?>)
+            {
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               // for non-formatting actions, do not apply the behavior IF it is a field-display-behavior //
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               applyBehavior = false;
+            }
+
+            if(applyBehavior)
+            {
+               fieldBehavior.apply(action, recordList, instance, table, field);
+            }
          }
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java

@@ -149,8 +149,7 @@ public class QInstanceHelpContentManager
          }
          else if(StringUtils.hasContent(widgetName))
          {
-            processHelpContentForWidget(key, widgetName, slotName, helpContent);
-
+            processHelpContentForWidget(key, widgetName, slotName, roles, helpContent);
          }
       }
       catch(Exception e)
@@ -252,7 +251,7 @@ public class QInstanceHelpContentManager
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void processHelpContentForWidget(String key, String widgetName, String slotName, QHelpContent helpContent)
+   private static void processHelpContentForWidget(String key, String widgetName, String slotName, Set<HelpRole> roles, QHelpContent helpContent)
    {
       QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(widgetName);
       if(!StringUtils.hasContent(slotName))
@@ -265,22 +264,14 @@ public class QInstanceHelpContentManager
       }
       else
       {
-         Map<String, QHelpContent> widgetHelpContent = widget.getHelpContent();
-         if(widgetHelpContent == null)
-         {
-            widgetHelpContent = new HashMap<>();
-         }
-
          if(helpContent != null)
          {
-            widgetHelpContent.put(slotName, helpContent);
+            widget.withHelpContent(slotName, helpContent);
          }
          else
          {
-            widgetHelpContent.remove(slotName);
+            widget.removeHelpContent(slotName, roles);
          }
-
-         widget.setHelpContent(widgetHelpContent);
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -64,6 +64,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.ParentWidgetMetaD
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
@@ -85,6 +86,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript;
@@ -623,6 +625,11 @@ public class QInstanceValidator
                supplementalTableMetaData.validate(qInstance, table, this);
             }
 
+            if(table.getShareableTableMetaData() != null)
+            {
+               table.getShareableTableMetaData().validate(qInstance, table, this);
+            }
+
             runPlugins(QTableMetaData.class, table, qInstance);
          });
       }
@@ -710,7 +717,6 @@ public class QInstanceValidator
    {
       String prefix = "Table " + table.getName() + " ";
 
-      RECORD_SECURITY_LOCKS_LOOP:
       for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
       {
          if(!assertCondition(recordSecurityLock != null, prefix + "has a null recordSecurityLock (did you mean to give it a null list of locks?)"))
@@ -718,90 +724,14 @@ public class QInstanceValidator
             continue;
          }
 
-         String securityKeyTypeName = recordSecurityLock.getSecurityKeyType();
-         if(assertCondition(StringUtils.hasContent(securityKeyTypeName), prefix + "has a recordSecurityLock that is missing a securityKeyType"))
+         if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock)
          {
-            assertCondition(qInstance.getSecurityKeyType(securityKeyTypeName) != null, prefix + "has a recordSecurityLock with an unrecognized securityKeyType: " + securityKeyTypeName);
+            validateMultiRecordSecurityLock(qInstance, table, multiRecordSecurityLock, prefix);
          }
-
-         prefix = "Table " + table.getName() + " recordSecurityLock (of key type " + securityKeyTypeName + ") ";
-
-         assertCondition(recordSecurityLock.getLockScope() != null, prefix + " is missing its lockScope");
-
-         boolean hasAnyBadJoins = false;
-         for(String joinName : CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()))
+         else
          {
-            if(!assertCondition(qInstance.getJoin(joinName) != null, prefix + "has an unrecognized joinName: " + joinName))
-            {
-               hasAnyBadJoins = true;
-            }
+            validateRecordSecurityLock(qInstance, table, recordSecurityLock, prefix);
          }
-
-         String fieldName = recordSecurityLock.getFieldName();
-
-         ////////////////////////////////////////////////////////////////////////////////
-         // don't bother trying to validate field names if we know we have a bad join. //
-         ////////////////////////////////////////////////////////////////////////////////
-         if(assertCondition(StringUtils.hasContent(fieldName), prefix + "is missing a fieldName") && !hasAnyBadJoins)
-         {
-            if(fieldName.contains("."))
-            {
-               if(assertCondition(CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain()), prefix + "field name " + fieldName + " looks like a join (has a dot), but no joinNameChain was given."))
-               {
-                  List<QueryJoin> joins = new ArrayList<>();
-
-                  ///////////////////////////////////////////////////////////////////////////////////////////////////
-                  // ok - so - the join name chain is going to be like this:                                       //
-                  // for a table:  orderLineItemExtrinsic (that's 2 away from order, where the security field is): //
-                  // - securityFieldName = order.clientId                                                          //
-                  // - joinNameChain = orderJoinOrderLineItem, orderLineItemJoinOrderLineItemExtrinsic             //
-                  // so - to navigate from the table to the security field, we need to reverse the joinNameChain,  //
-                  // and step (via tmpTable variable) back to the securityField                                    //
-                  ///////////////////////////////////////////////////////////////////////////////////////////////////
-                  ArrayList<String> joinNameChain = new ArrayList<>(CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()));
-                  Collections.reverse(joinNameChain);
-
-                  QTableMetaData tmpTable = table;
-
-                  for(String joinName : joinNameChain)
-                  {
-                     QJoinMetaData join = qInstance.getJoin(joinName);
-                     if(join == null)
-                     {
-                        errors.add(prefix + "joinNameChain contained an unrecognized join: " + joinName);
-                        continue RECORD_SECURITY_LOCKS_LOOP;
-                     }
-
-                     if(join.getLeftTable().equals(tmpTable.getName()))
-                     {
-                        joins.add(new QueryJoin(join));
-                        tmpTable = qInstance.getTable(join.getRightTable());
-                     }
-                     else if(join.getRightTable().equals(tmpTable.getName()))
-                     {
-                        joins.add(new QueryJoin(join.flip()));
-                        tmpTable = qInstance.getTable(join.getLeftTable());
-                     }
-                     else
-                     {
-                        errors.add(prefix + "joinNameChain could not be followed through join: " + joinName);
-                        continue RECORD_SECURITY_LOCKS_LOOP;
-                     }
-                  }
-
-                  assertCondition(findField(qInstance, table, joins, fieldName), prefix + "has an unrecognized fieldName: " + fieldName);
-               }
-            }
-            else
-            {
-               if(assertCondition(CollectionUtils.nullSafeIsEmpty(recordSecurityLock.getJoinNameChain()), prefix + "field name " + fieldName + " does not look like a join (does not have a dot), but a joinNameChain was given."))
-               {
-                  assertNoException(() -> table.getField(fieldName), prefix + "has an unrecognized fieldName: " + fieldName);
-               }
-            }
-         }
-
-         assertCondition(recordSecurityLock.getNullValueBehavior() != null, prefix + "is missing a nullValueBehavior");
       }
    }
 
@@ -810,7 +740,121 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void validateTableField(QInstance qInstance, String tableName, String fieldName, QTableMetaData table, QFieldMetaData field)
+   private void validateMultiRecordSecurityLock(QInstance qInstance, QTableMetaData table, MultiRecordSecurityLock multiRecordSecurityLock, String prefix)
+   {
+      assertCondition(multiRecordSecurityLock.getOperator() != null, prefix + "has a MultiRecordSecurityLock that is missing an operator");
+
+      for(RecordSecurityLock lock : multiRecordSecurityLock.getLocks())
+      {
+         validateRecordSecurityLock(qInstance, table, lock, prefix);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void validateRecordSecurityLock(QInstance qInstance, QTableMetaData table, RecordSecurityLock recordSecurityLock, String prefix)
+   {
+      String securityKeyTypeName = recordSecurityLock.getSecurityKeyType();
+      if(assertCondition(StringUtils.hasContent(securityKeyTypeName), prefix + "has a recordSecurityLock that is missing a securityKeyType"))
+      {
+         assertCondition(qInstance.getSecurityKeyType(securityKeyTypeName) != null, prefix + "has a recordSecurityLock with an unrecognized securityKeyType: " + securityKeyTypeName);
+      }
+
+      prefix = "Table " + table.getName() + " recordSecurityLock (of key type " + securityKeyTypeName + ") ";
+
+      assertCondition(recordSecurityLock.getLockScope() != null, prefix + " is missing its lockScope");
+
+      boolean hasAnyBadJoins = false;
+      for(String joinName : CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()))
+      {
+         if(!assertCondition(qInstance.getJoin(joinName) != null, prefix + "has an unrecognized joinName: " + joinName))
+         {
+            hasAnyBadJoins = true;
+         }
+      }
+
+      String fieldName = recordSecurityLock.getFieldName();
+
+      ////////////////////////////////////////////////////////////////////////////////
+      // don't bother trying to validate field names if we know we have a bad join. //
+      ////////////////////////////////////////////////////////////////////////////////
+      if(assertCondition(StringUtils.hasContent(fieldName), prefix + "is missing a fieldName") && !hasAnyBadJoins)
+      {
+         if(fieldName.contains("."))
+         {
+            if(assertCondition(CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain()), prefix + "field name " + fieldName + " looks like a join (has a dot), but no joinNameChain was given."))
+            {
+               String[] split         = fieldName.split("\\.");
+               String   joinTableName = split[0];
+               String   joinFieldName = split[1];
+
+               List<QueryJoin> joins = new ArrayList<>();
+
+               ///////////////////////////////////////////////////////////////////////////////////////////////////
+               // ok - so - the join name chain is going to be like this:                                       //
+               // for a table:  orderLineItemExtrinsic (that's 2 away from order, where the security field is): //
+               // - securityFieldName = order.clientId                                                          //
+               // - joinNameChain = orderJoinOrderLineItem, orderLineItemJoinOrderLineItemExtrinsic             //
+               // so - to navigate from the table to the security field, we need to reverse the joinNameChain,  //
+               // and step (via tmpTable variable) back to the securityField                                    //
+               ///////////////////////////////////////////////////////////////////////////////////////////////////
+               ArrayList<String> joinNameChain = new ArrayList<>(CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()));
+               Collections.reverse(joinNameChain);
+
+               QTableMetaData tmpTable = table;
+
+               for(String joinName : joinNameChain)
+               {
+                  QJoinMetaData join = qInstance.getJoin(joinName);
+                  if(join == null)
+                  {
+                     errors.add(prefix + "joinNameChain contained an unrecognized join: " + joinName);
+                     return;
+                  }
+
+                  if(join.getLeftTable().equals(tmpTable.getName()))
+                  {
+                     joins.add(new QueryJoin(join));
+                     tmpTable = qInstance.getTable(join.getRightTable());
+                  }
+                  else if(join.getRightTable().equals(tmpTable.getName()))
+                  {
+                     joins.add(new QueryJoin(join.flip()));
+                     tmpTable = qInstance.getTable(join.getLeftTable());
+                  }
+                  else
+                  {
+                     errors.add(prefix + "joinNameChain could not be followed through join: " + joinName);
+                     return;
+                  }
+               }
+
+               assertCondition(Objects.equals(tmpTable.getName(), joinTableName), prefix + "has a joinNameChain doesn't end in the expected table [" + joinTableName + "] (was: " + tmpTable.getName() + ")");
+
+               assertCondition(findField(qInstance, table, joins, fieldName), prefix + "has an unrecognized fieldName: " + fieldName);
+            }
+         }
+         else
+         {
+            if(assertCondition(CollectionUtils.nullSafeIsEmpty(recordSecurityLock.getJoinNameChain()), prefix + "field name " + fieldName + " does not look like a join (does not have a dot), but a joinNameChain was given."))
+            {
+               assertNoException(() -> table.getField(fieldName), prefix + "has an unrecognized fieldName: " + fieldName);
+            }
+         }
+      }
+
+      assertCondition(recordSecurityLock.getNullValueBehavior() != null, prefix + "is missing a nullValueBehavior");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private <T extends FieldBehavior<T>> void validateTableField(QInstance qInstance, String tableName, String fieldName, QTableMetaData table, QFieldMetaData field)
    {
       assertCondition(Objects.equals(fieldName, field.getName()),
          "Inconsistent naming in table " + tableName + " for field " + fieldName + "/" + field.getName() + ".");
@@ -823,12 +867,32 @@ public class QInstanceValidator
 
       String prefix = "Field " + fieldName + " in table " + tableName + " ";
 
+      ///////////////////////////////////////////////////
+      // validate things we know about field behaviors //
+      ///////////////////////////////////////////////////
       ValueTooLongBehavior behavior = field.getBehaviorOrDefault(qInstance, ValueTooLongBehavior.class);
       if(behavior != null && !behavior.equals(ValueTooLongBehavior.PASS_THROUGH))
       {
          assertCondition(field.getMaxLength() != null, prefix + "specifies a ValueTooLongBehavior, but not a maxLength.");
       }
 
+      Set<Class<FieldBehavior<T>>> usedFieldBehaviorTypes = new HashSet<>();
+      if(field.getBehaviors() != null)
+      {
+         for(FieldBehavior<?> fieldBehavior : field.getBehaviors())
+         {
+            Class<FieldBehavior<T>> behaviorClass = (Class<FieldBehavior<T>>) fieldBehavior.getClass();
+
+            errors.addAll(fieldBehavior.validateBehaviorConfiguration(table, field));
+
+            if(!fieldBehavior.allowMultipleBehaviorsOfThisType())
+            {
+               assertCondition(!usedFieldBehaviorTypes.contains(behaviorClass), prefix + "has more than 1 fieldBehavior of type " + behaviorClass.getSimpleName() + ", which is not allowed for this type");
+            }
+            usedFieldBehaviorTypes.add(behaviorClass);
+         }
+      }
+
       if(field.getMaxLength() != null)
       {
          assertCondition(field.getMaxLength() > 0, prefix + "has an invalid maxLength (" + field.getMaxLength() + ") - must be greater than 0.");
@@ -1449,7 +1513,7 @@ public class QInstanceValidator
    private void validateScheduleMetaData(QScheduleMetaData schedule, QInstance qInstance, String prefix)
    {
       boolean isRepeat = schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null;
-      boolean isCron = StringUtils.hasContent(schedule.getCronExpression());
+      boolean isCron   = StringUtils.hasContent(schedule.getCronExpression());
       assertCondition(isRepeat || isCron, prefix + " either repeatMillis or repeatSeconds or cronExpression must be set");
       assertCondition(!(isRepeat && isCron), prefix + " both a repeat time and cronExpression may not be set");
 
@@ -1469,8 +1533,8 @@ public class QInstanceValidator
 
          if(assertCondition(StringUtils.hasContent(schedule.getCronTimeZoneId()), prefix + " a cron schedule must specify a cronTimeZoneId"))
          {
-            String[] availableIDs = TimeZone.getAvailableIDs();
-            Optional<String> first = Arrays.stream(availableIDs).filter(id -> id.equals(schedule.getCronTimeZoneId())).findFirst();
+            String[]         availableIDs = TimeZone.getAvailableIDs();
+            Optional<String> first        = Arrays.stream(availableIDs).filter(id -> id.equals(schedule.getCronTimeZoneId())).findFirst();
             assertCondition(first.isPresent(), prefix + " unrecognized cronTimeZoneId: " + schedule.getCronTimeZoneId());
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java

@@ -26,6 +26,7 @@ import java.util.UUID;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
+import com.kingsrook.qqq.backend.core.actions.async.NonPersistedAsyncJobCallback;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
 import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
@@ -139,7 +140,7 @@ public class AbstractActionInput
          ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
          // don't return null here (too easy to NPE).  instead, if someone wants one of these, create one and give it to them. //
          ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         asyncJobCallback = new AsyncJobCallback(UUID.randomUUID(), new AsyncJobStatus());
+         asyncJobCallback = new NonPersistedAsyncJobCallback(UUID.randomUUID(), new AsyncJobStatus().withJobName(getClass().getSimpleName()));
       }
       return asyncJobCallback;
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java

@@ -30,6 +30,7 @@ import java.util.Map;
 import java.util.UUID;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
+import com.kingsrook.qqq.backend.core.actions.async.NonPersistedAsyncJobCallback;
 import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
@@ -450,7 +451,7 @@ public class RunBackendStepInput extends AbstractActionInput
          /////////////////////////////////////////////////////////////////////////
          // avoid NPE in case we didn't have one of these!  create a new one... //
          /////////////////////////////////////////////////////////////////////////
-         asyncJobCallback = new AsyncJobCallback(UUID.randomUUID(), new AsyncJobStatus());
+         asyncJobCallback = new NonPersistedAsyncJobCallback(UUID.randomUUID(), new AsyncJobStatus().withJobName(processName + "." + stepName));
       }
       return (asyncJobCallback);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportDestination.java

@@ -26,7 +26,8 @@ import java.io.OutputStream;
 
 
 /*******************************************************************************
- **
+ ** Member of report & export Inputs, that wraps details about the destination of
+ ** where & how the report (or export) is being written.
  *******************************************************************************/
 public class ReportDestination
 {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormatPossibleValueEnum.java

@@ -31,8 +31,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValu
 public enum ReportFormatPossibleValueEnum implements PossibleValueEnum<String>
 {
    XLSX,
-   JSON,
-   CSV;
+   CSV,
+   JSON;
 
    public static final String NAME = "reportFormat";
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableDefinition.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable;
 
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -29,7 +30,7 @@ import java.util.List;
 /*******************************************************************************
  ** Full definition of a pivot table - its rows, columns, and values.
  *******************************************************************************/
-public class PivotTableDefinition implements Cloneable
+public class PivotTableDefinition implements Cloneable, Serializable
 {
    private List<PivotTableGroupBy> rows;
    private List<PivotTableGroupBy> columns;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableGroupBy.java

@@ -22,11 +22,14 @@
 package com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable;
 
 
+import java.io.Serializable;
+
+
 /*******************************************************************************
  ** Either a row or column grouping in a pivot table.  e.g., a field plus
  ** sorting details, plus showTotals boolean.
  *******************************************************************************/
-public class PivotTableGroupBy implements Cloneable
+public class PivotTableGroupBy implements Cloneable, Serializable
 {
    private String            fieldName;
    private PivotTableOrderBy orderBy;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableOrderBy.java

@@ -22,10 +22,13 @@
 package com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable;
 
 
+import java.io.Serializable;
+
+
 /*******************************************************************************
  ** How a group-by (rows or columns) should be sorted.
  *******************************************************************************/
-public class PivotTableOrderBy
+public class PivotTableOrderBy implements Serializable
 {
    // todo - implement, but only if POI supports (or we build our own support...)
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableValue.java

@@ -22,10 +22,13 @@
 package com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable;
 
 
+import java.io.Serializable;
+
+
 /*******************************************************************************
  ** a value (e.g., field name + function) used in a pivot table
  *******************************************************************************/
-public class PivotTableValue implements Cloneable
+public class PivotTableValue implements Cloneable, Serializable
 {
    private String             fieldName;
    private PivotTableFunction function;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.delete;
 
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
@@ -139,6 +140,24 @@ public class DeleteInput extends AbstractTableActionInput
 
 
 
+   /*******************************************************************************
+    ** Fluently add 1 primary key to the delete input
+    **
+    *******************************************************************************/
+   public DeleteInput withPrimaryKey(Serializable primaryKey)
+   {
+      if(primaryKeys == null)
+      {
+         primaryKeys = new ArrayList<>();
+      }
+
+      primaryKeys.add(primaryKey);
+
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     ** Setter for ids
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.model.actions.tables.query;
 
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -37,12 +38,17 @@ import com.kingsrook.qqq.backend.core.logging.LogPair;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.MutableList;
 import org.apache.logging.log4j.Level;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -60,11 +66,23 @@ public class JoinsContext
    private final String          mainTableName;
    private final List<QueryJoin> queryJoins;
 
+   private final QQueryFilter securityFilter;
+
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   // pointer either at securityFilter, or at a sub-filter within it, for when we're doing a recursive build-out of multi-locks //
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   private QQueryFilter securityFilterCursor;
+
    ////////////////////////////////////////////////////////////////
    // note - will have entries for all tables, not just aliases. //
    ////////////////////////////////////////////////////////////////
    private final Map<String, String> aliasToTableNameMap = new HashMap<>();
-   private       Level               logLevel            = Level.OFF;
+
+   /////////////////////////////////////////////////////////////////////////////
+   // we will get a TON of more output if this gets turned up, so be cautious //
+   /////////////////////////////////////////////////////////////////////////////
+   private Level logLevel = Level.OFF;
+   private Level logLevelForFilter = Level.OFF;
 
 
 
@@ -74,54 +92,225 @@ public class JoinsContext
     *******************************************************************************/
    public JoinsContext(QInstance instance, String tableName, List<QueryJoin> queryJoins, QQueryFilter filter) throws QException
    {
-      log("--- START ----------------------------------------------------------------------", logPair("mainTable", tableName));
       this.instance = instance;
       this.mainTableName = tableName;
       this.queryJoins = new MutableList<>(queryJoins);
+      this.securityFilter = new QQueryFilter();
+      this.securityFilterCursor = this.securityFilter;
+
+      // log("--- START ----------------------------------------------------------------------", logPair("mainTable", tableName));
+      dumpDebug(true, false);
 
       for(QueryJoin queryJoin : this.queryJoins)
       {
-         log("Processing input query join", logPair("joinTable", queryJoin.getJoinTable()), logPair("alias", queryJoin.getAlias()), logPair("baseTableOrAlias", queryJoin.getBaseTableOrAlias()), logPair("joinMetaDataName", () -> queryJoin.getJoinMetaData().getName()));
          processQueryJoin(queryJoin);
       }
 
+      /////////////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure that all tables specified in filter columns are being brought into the query as joins //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////
+      ensureFilterIsRepresented(filter);
+      logFilter("After ensureFilterIsRepresented:", securityFilter);
+
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // ensure that any record locks on the main table, which require a join, are present //
+      ///////////////////////////////////////////////////////////////////////////////////////
+      MultiRecordSecurityLock multiRecordSecurityLock = RecordSecurityLockFilters.filterForReadLockTree(CollectionUtils.nonNullList(instance.getTable(tableName).getRecordSecurityLocks()));
+      for(RecordSecurityLock lock : multiRecordSecurityLock.getLocks())
+      {
+         ensureRecordSecurityLockIsRepresented(tableName, tableName, lock, null);
+         logFilter("After ensureRecordSecurityLockIsRepresented[fieldName=" + lock.getFieldName() + "]:", securityFilter);
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////
+      // make sure that all joins in the query have meta data specified                //
+      // e.g., a user-added join may just specify the join-table                       //
+      // or a join implicitly added from a filter may also not have its join meta data //
+      ///////////////////////////////////////////////////////////////////////////////////
+      fillInMissingJoinMetaData();
+      logFilter("After fillInMissingJoinMetaData:", securityFilter);
+
       ///////////////////////////////////////////////////////////////
       // ensure any joins that contribute a recordLock are present //
       ///////////////////////////////////////////////////////////////
-      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(instance.getTable(tableName).getRecordSecurityLocks())))
-      {
-         ensureRecordSecurityLockIsRepresented(instance, tableName, recordSecurityLock);
-      }
+      ensureAllJoinRecordSecurityLocksAreRepresented(instance);
+      logFilter("After ensureAllJoinRecordSecurityLocksAreRepresented:", securityFilter);
 
-      ensureFilterIsRepresented(filter);
-
-      addJoinsFromExposedJoinPaths();
-
-      /* todo!!
-      for(QueryJoin queryJoin : queryJoins)
-      {
-         QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable());
-         for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(joinTable.getRecordSecurityLocks()))
-         {
-            // addCriteriaForRecordSecurityLock(instance, session, joinTable, securityCriteria, recordSecurityLock, joinsContext, queryJoin.getJoinTableOrItsAlias());
-         }
-      }
-      */
+      ////////////////////////////////////////////////////////////////////////////////////
+      // if there were any security filters built, then put those into the input filter //
+      ////////////////////////////////////////////////////////////////////////////////////
+      addSecurityFiltersToInputFilter(filter);
 
       log("Constructed JoinsContext", logPair("mainTableName", this.mainTableName), logPair("queryJoins", this.queryJoins.stream().map(qj -> qj.getJoinTable()).collect(Collectors.joining(","))));
-      log("--- END ------------------------------------------------------------------------");
+      log("", logPair("securityFilter", securityFilter));
+      log("", logPair("fullFilter", filter));
+      dumpDebug(false, true);
+      // log("--- END ------------------------------------------------------------------------");
    }
 
 
 
    /*******************************************************************************
-    **
+    ** Update the input filter with any security filters that were built.
     *******************************************************************************/
-   private void ensureRecordSecurityLockIsRepresented(QInstance instance, String tableName, RecordSecurityLock recordSecurityLock) throws QException
+   private void addSecurityFiltersToInputFilter(QQueryFilter filter)
    {
+      ////////////////////////////////////////////////////////////////////////////////////
+      // if there's no security filter criteria (including sub-filters), return w/ noop //
+      ////////////////////////////////////////////////////////////////////////////////////
+      if(CollectionUtils.nullSafeIsEmpty(securityFilter.getSubFilters()))
+      {
+         return;
+      }
+
+      ///////////////////////////////////////////////////////////////////////
+      // if the input filter is an OR we need to replace it with a new AND //
+      ///////////////////////////////////////////////////////////////////////
+      if(filter.getBooleanOperator().equals(QQueryFilter.BooleanOperator.OR))
+      {
+         List<QFilterCriteria> originalCriteria   = filter.getCriteria();
+         List<QQueryFilter>    originalSubFilters = filter.getSubFilters();
+
+         QQueryFilter replacementFilter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
+         replacementFilter.setCriteria(originalCriteria);
+         replacementFilter.setSubFilters(originalSubFilters);
+
+         filter.setCriteria(new ArrayList<>());
+         filter.setSubFilters(new ArrayList<>());
+         filter.setBooleanOperator(QQueryFilter.BooleanOperator.AND);
+         filter.addSubFilter(replacementFilter);
+      }
+
+      filter.addSubFilter(securityFilter);
+   }
+
+
+
+   /*******************************************************************************
+    ** In case we've added any joins to the query that have security locks which
+    ** weren't previously added to the query, add them now.  basically, this is
+    ** calling ensureRecordSecurityLockIsRepresented for each queryJoin.
+    *******************************************************************************/
+   private void ensureAllJoinRecordSecurityLocksAreRepresented(QInstance instance) throws QException
+   {
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // avoid concurrent modification exceptions by doing a double-loop and breaking the inner any time anything gets added //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      Set<QueryJoin> processedQueryJoins   = new HashSet<>();
+      boolean        addedAnyThisIteration = true;
+      while(addedAnyThisIteration)
+      {
+         addedAnyThisIteration = false;
+
+         for(QueryJoin queryJoin : this.queryJoins)
+         {
+            boolean addedAnyForThisJoin = false;
+
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // avoid double-processing the same query join                                                                                            //
+            // or adding security filters for a join who was only added to the query so that we could add locks (an ImplicitQueryJoinForSecurityLock) //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(processedQueryJoins.contains(queryJoin) || queryJoin instanceof ImplicitQueryJoinForSecurityLock)
+            {
+               continue;
+            }
+            processedQueryJoins.add(queryJoin);
+
+            //////////////////////////////////////////////////////////////////////////////////////////
+            // process all locks on this join's join-table.  keep track if any new joins were added //
+            //////////////////////////////////////////////////////////////////////////////////////////
+            QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable());
+
+            MultiRecordSecurityLock multiRecordSecurityLock = RecordSecurityLockFilters.filterForReadLockTree(CollectionUtils.nonNullList(joinTable.getRecordSecurityLocks()));
+            for(RecordSecurityLock lock : multiRecordSecurityLock.getLocks())
+            {
+               List<QueryJoin> addedQueryJoins = ensureRecordSecurityLockIsRepresented(joinTable.getName(), queryJoin.getJoinTableOrItsAlias(), lock, queryJoin);
+
+               //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // if any joins were added by this call, add them to the set of processed ones, so they don't get re-processed. //
+               // also mark the flag that any were added for this join, to manage the double-looping                           //
+               //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(CollectionUtils.nullSafeHasContents(addedQueryJoins))
+               {
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // make all new joins added in that method be of the same type (inner/left/etc) as the query join they are connected to //
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  for(QueryJoin addedQueryJoin : addedQueryJoins)
+                  {
+                     addedQueryJoin.setType(queryJoin.getType());
+                  }
+
+                  processedQueryJoins.addAll(addedQueryJoins);
+                  addedAnyForThisJoin = true;
+               }
+            }
+
+            ///////////////////////////////////////////////////////////////////////////////////////////////
+            // if any new joins were added, we need to break the inner-loop, and continue the outer loop //
+            // e.g., to process the next query join (but we can't just go back to the foreach queryJoin, //
+            // because it would fail with concurrent modification)                                       //
+            ///////////////////////////////////////////////////////////////////////////////////////////////
+            if(addedAnyForThisJoin)
+            {
+               addedAnyThisIteration = true;
+               break;
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** For a given recordSecurityLock on a given table (with a possible alias),
+    ** make sure that if any joins are needed to get to the lock, that they are in the query.
+    **
+    ** returns the list of query joins that were added, if any were added
+    *******************************************************************************/
+   private List<QueryJoin> ensureRecordSecurityLockIsRepresented(String tableName, String tableNameOrAlias, RecordSecurityLock recordSecurityLock, QueryJoin sourceQueryJoin) throws QException
+   {
+      List<QueryJoin> addedQueryJoins = new ArrayList<>();
+
+      ////////////////////////////////////////////////////////////////////////////
+      // if this lock is a multi-lock, then recursively process its child-locks //
+      ////////////////////////////////////////////////////////////////////////////
+      if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock)
+      {
+         log("Processing MultiRecordSecurityLock...");
+
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // make a new level in the filter-tree - storing old cursor, and updating cursor to point at new level //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////
+         QQueryFilter oldSecurityFilterCursor = this.securityFilterCursor;
+         QQueryFilter nextLevelSecurityFilter = new QQueryFilter();
+         this.securityFilterCursor.addSubFilter(nextLevelSecurityFilter);
+         this.securityFilterCursor = nextLevelSecurityFilter;
+
+         ///////////////////////////////////////
+         // set the boolean operator to match //
+         ///////////////////////////////////////
+         nextLevelSecurityFilter.setBooleanOperator(multiRecordSecurityLock.getOperator().toFilterOperator());
+
+         //////////////////////
+         // process children //
+         //////////////////////
+         for(RecordSecurityLock childLock : CollectionUtils.nonNullList(multiRecordSecurityLock.getLocks()))
+         {
+            log(" - Recursive call for childLock: " + childLock);
+            addedQueryJoins.addAll(ensureRecordSecurityLockIsRepresented(tableName, tableNameOrAlias, childLock, sourceQueryJoin));
+         }
+
+         ////////////////////
+         // restore cursor //
+         ////////////////////
+         this.securityFilterCursor = oldSecurityFilterCursor;
+
+         return addedQueryJoins;
+      }
+
       ///////////////////////////////////////////////////////////////////////////////////////////////////
-      // ok - so - the join name chain is going to be like this:                                       //
-      // for a table:  orderLineItemExtrinsic (that's 2 away from order, where the security field is): //
+      // A join name chain is going to look like this:                                                 //
+      // for a table:  orderLineItemExtrinsic (that's 2 away from order, where its security field is): //
       // - securityFieldName = order.clientId                                                          //
       // - joinNameChain = orderJoinOrderLineItem, orderLineItemJoinOrderLineItemExtrinsic             //
       // so - to navigate from the table to the security field, we need to reverse the joinNameChain,  //
@@ -129,30 +318,30 @@ public class JoinsContext
       ///////////////////////////////////////////////////////////////////////////////////////////////////
       ArrayList<String> joinNameChain = new ArrayList<>(CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()));
       Collections.reverse(joinNameChain);
-      log("Evaluating recordSecurityLock", logPair("recordSecurityLock", recordSecurityLock.getFieldName()), logPair("joinNameChain", joinNameChain));
+      log("Evaluating recordSecurityLock.  Join name chain is of length: " + joinNameChain.size(), logPair("tableNameOrAlias", tableNameOrAlias), logPair("recordSecurityLock", recordSecurityLock.getFieldName()), logPair("joinNameChain", joinNameChain));
 
-      QTableMetaData tmpTable = instance.getTable(mainTableName);
+      QTableMetaData tmpTable                = instance.getTable(tableName);
+      String         securityFieldTableAlias = tableNameOrAlias;
+      String         baseTableOrAlias        = tableNameOrAlias;
+
+      boolean chainIsInner = true;
+      if(sourceQueryJoin != null && QueryJoin.Type.isOuter(sourceQueryJoin.getType()))
+      {
+         chainIsInner = false;
+      }
 
       for(String joinName : joinNameChain)
       {
-         ///////////////////////////////////////////////////////////////////////////////////////////////////////
-         // check the joins currently in the query - if any are for this table, then we don't need to add one //
-         ///////////////////////////////////////////////////////////////////////////////////////////////////////
-         List<QueryJoin> matchingJoins = this.queryJoins.stream().filter(queryJoin ->
+         //////////////////////////////////////////////////////////////////////////////////////////////////
+         // check the joins currently in the query - if any are THIS join, then we don't need to add one //
+         //////////////////////////////////////////////////////////////////////////////////////////////////
+         List<QueryJoin> matchingQueryJoins = this.queryJoins.stream().filter(queryJoin ->
          {
-            QJoinMetaData joinMetaData = null;
-            if(queryJoin.getJoinMetaData() != null)
-            {
-               joinMetaData = queryJoin.getJoinMetaData();
-            }
-            else
-            {
-               joinMetaData = findJoinMetaData(instance, tableName, queryJoin.getJoinTable());
-            }
+            QJoinMetaData joinMetaData = queryJoin.getJoinMetaData();
             return (joinMetaData != null && Objects.equals(joinMetaData.getName(), joinName));
          }).toList();
 
-         if(CollectionUtils.nullSafeHasContents(matchingJoins))
+         if(CollectionUtils.nullSafeHasContents(matchingQueryJoins))
          {
             /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             // note - if a user added a join as an outer type, we need to change it to be inner, for the security purpose. //
@@ -160,11 +349,40 @@ public class JoinsContext
             /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             log("- skipping join already in the query", logPair("joinName", joinName));
 
-            if(matchingJoins.get(0).getType().equals(QueryJoin.Type.LEFT) || matchingJoins.get(0).getType().equals(QueryJoin.Type.RIGHT))
+            QueryJoin matchedQueryJoin = matchingQueryJoins.get(0);
+
+            if(matchedQueryJoin.getType().equals(QueryJoin.Type.LEFT) || matchedQueryJoin.getType().equals(QueryJoin.Type.RIGHT))
+            {
+               chainIsInner = false;
+            }
+
+            /* ?? todo ??
+            if(matchedQueryJoin.getType().equals(QueryJoin.Type.LEFT) || matchedQueryJoin.getType().equals(QueryJoin.Type.RIGHT))
             {
                log("- - although... it was here as an outer - so switching it to INNER", logPair("joinName", joinName));
-               matchingJoins.get(0).setType(QueryJoin.Type.INNER);
+               matchedQueryJoin.setType(QueryJoin.Type.INNER);
             }
+            */
+
+            //////////////////////////////////////////////////////////////////////////////////////////////////////
+            // as we're walking from tmpTable to the table which ultimately has the security key field,         //
+            // if the queryJoin we just found is joining out to tmpTable, then we need to advance tmpTable back //
+            // to the queryJoin's base table - else, tmpTable advances to the matched queryJoin's joinTable     //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(tmpTable.getName().equals(matchedQueryJoin.getJoinTable()))
+            {
+               securityFieldTableAlias = Objects.requireNonNullElse(matchedQueryJoin.getBaseTableOrAlias(), mainTableName);
+            }
+            else
+            {
+               securityFieldTableAlias = matchedQueryJoin.getJoinTableOrItsAlias();
+            }
+            tmpTable = instance.getTable(securityFieldTableAlias);
+
+            ////////////////////////////////////////////////////////////////////////////////////////
+            // set the baseTableOrAlias for the next iteration to be this join's joinTableOrAlias //
+            ////////////////////////////////////////////////////////////////////////////////////////
+            baseTableOrAlias = securityFieldTableAlias;
 
             continue;
          }
@@ -172,20 +390,233 @@ public class JoinsContext
          QJoinMetaData join = instance.getJoin(joinName);
          if(join.getLeftTable().equals(tmpTable.getName()))
          {
-            QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join).withType(QueryJoin.Type.INNER);
-            this.addQueryJoin(queryJoin, "forRecordSecurityLock (non-flipped)");
+            securityFieldTableAlias = join.getRightTable() + "_forSecurityJoin_" + join.getName();
+            QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock()
+               .withJoinMetaData(join)
+               .withType(chainIsInner ? QueryJoin.Type.INNER : QueryJoin.Type.LEFT)
+               .withBaseTableOrAlias(baseTableOrAlias)
+               .withAlias(securityFieldTableAlias);
+
+            if(securityFilterCursor.getBooleanOperator() == QQueryFilter.BooleanOperator.OR)
+            {
+               queryJoin.withType(QueryJoin.Type.LEFT);
+               chainIsInner = false;
+            }
+
+            addQueryJoin(queryJoin, "forRecordSecurityLock (non-flipped)", "- ");
+            addedQueryJoins.add(queryJoin);
             tmpTable = instance.getTable(join.getRightTable());
          }
          else if(join.getRightTable().equals(tmpTable.getName()))
          {
-            QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join.flip()).withType(QueryJoin.Type.INNER);
-            this.addQueryJoin(queryJoin, "forRecordSecurityLock (flipped)");
+            securityFieldTableAlias = join.getLeftTable() + "_forSecurityJoin_" + join.getName();
+            QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock()
+               .withJoinMetaData(join.flip())
+               .withType(chainIsInner ? QueryJoin.Type.INNER : QueryJoin.Type.LEFT)
+               .withBaseTableOrAlias(baseTableOrAlias)
+               .withAlias(securityFieldTableAlias);
+
+            if(securityFilterCursor.getBooleanOperator() == QQueryFilter.BooleanOperator.OR)
+            {
+               queryJoin.withType(QueryJoin.Type.LEFT);
+               chainIsInner = false;
+            }
+
+            addQueryJoin(queryJoin, "forRecordSecurityLock (flipped)", "- ");
+            addedQueryJoins.add(queryJoin);
             tmpTable = instance.getTable(join.getLeftTable());
          }
          else
          {
+            dumpDebug(false, true);
             throw (new QException("Error adding security lock joins to query - table name [" + tmpTable.getName() + "] not found in join [" + joinName + "]"));
          }
+
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // for the next iteration of the loop, set the next join's baseTableOrAlias to be the alias we just created //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         baseTableOrAlias = securityFieldTableAlias;
+      }
+
+      ////////////////////////////////////////////////////////////////////////////////////
+      // now that we know the joins/tables are in the query, add to the security filter //
+      ////////////////////////////////////////////////////////////////////////////////////
+      QueryJoin lastAddedQueryJoin = addedQueryJoins.isEmpty() ? null : addedQueryJoins.get(addedQueryJoins.size() - 1);
+      if(sourceQueryJoin != null && lastAddedQueryJoin == null)
+      {
+         lastAddedQueryJoin = sourceQueryJoin;
+      }
+      addSubFilterForRecordSecurityLock(recordSecurityLock, tmpTable, securityFieldTableAlias, !chainIsInner, lastAddedQueryJoin);
+
+      log("Finished evaluating recordSecurityLock");
+
+      return (addedQueryJoins);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void addSubFilterForRecordSecurityLock(RecordSecurityLock recordSecurityLock, QTableMetaData table, String tableNameOrAlias, boolean isOuter, QueryJoin sourceQueryJoin)
+   {
+      QSession session = QContext.getQSession();
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // check if the key type has an all-access key, and if so, if it's set to true for the current user/session //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
+      boolean haveAllAccessKey = false;
+      if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()))
+      {
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if we have all-access on this key, then we don't need a criterion for it (as long as we're in an AND filter) //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(session.hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
+         {
+            haveAllAccessKey = true;
+
+            if(sourceQueryJoin != null)
+            {
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // in case the queryJoin object is re-used between queries, and its security criteria need to be different (!!), reset it //
+               // this can be exposed in tests - maybe not entirely expected in real-world, but seems safe enough                        //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               sourceQueryJoin.withSecurityCriteria(new ArrayList<>());
+            }
+
+            ////////////////////////////////////////////////////////////////////////////////////////
+            // if we're in an AND filter, then we don't need a criteria for this lock, so return. //
+            ////////////////////////////////////////////////////////////////////////////////////////
+            boolean inAnAndFilter = securityFilterCursor.getBooleanOperator() == QQueryFilter.BooleanOperator.AND;
+            if(inAnAndFilter)
+            {
+               return;
+            }
+         }
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////////
+      // for locks w/o a join chain, the lock fieldName will simply be a field on the table. //
+      // so just prepend that with the tableNameOrAlias.                                     //
+      /////////////////////////////////////////////////////////////////////////////////////////
+      String fieldName = tableNameOrAlias + "." + recordSecurityLock.getFieldName();
+      if(CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain()))
+      {
+         /////////////////////////////////////////////////////////////////////////////////
+         // else, expect a "table.field" in the lock fieldName - but we want to replace //
+         // the table name part with a possible alias that we took in.                  //
+         /////////////////////////////////////////////////////////////////////////////////
+         String[] parts = recordSecurityLock.getFieldName().split("\\.");
+         if(parts.length != 2)
+         {
+            dumpDebug(false, true);
+            throw new IllegalArgumentException("Mal-formatted recordSecurityLock fieldName for lock with joinNameChain in query: " + fieldName);
+         }
+         fieldName = tableNameOrAlias + "." + parts[1];
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      // else - get the key values from the session and decide what kind of criterion to build //
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      QQueryFilter          lockFilter   = new QQueryFilter();
+      List<QFilterCriteria> lockCriteria = new ArrayList<>();
+      lockFilter.setCriteria(lockCriteria);
+
+      QFieldType type = QFieldType.INTEGER;
+      try
+      {
+         JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = getFieldAndTableNameOrAlias(fieldName);
+         type = fieldAndTableNameOrAlias.field().getType();
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Error getting field type...  Trying Integer", e);
+      }
+
+      if(haveAllAccessKey)
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////
+         // if we have an all access key (but we got here because we're part of an OR query), then //
+         // write a criterion that will always be true - e.g., field=field                         //
+         ////////////////////////////////////////////////////////////////////////////////////////////
+         lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.TRUE));
+      }
+      else
+      {
+         List<Serializable> securityKeyValues = session.getSecurityKeyValues(recordSecurityLock.getSecurityKeyType(), type);
+         if(CollectionUtils.nullSafeIsEmpty(securityKeyValues))
+         {
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // handle user with no values -- they can only see null values, and only iff the lock's null-value behavior is ALLOW //
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(RecordSecurityLock.NullValueBehavior.ALLOW.equals(recordSecurityLock.getNullValueBehavior()))
+            {
+               lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK));
+            }
+            else
+            {
+               ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // else, if no user/session values, and null-value behavior is deny, then setup a FALSE condition, to allow no rows.         //
+               // todo - maybe avoid running the whole query - as you're not allowed ANY records (based on boolean tree down to this point) //
+               ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.FALSE));
+            }
+         }
+         else
+         {
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // else, if user/session has some values, build an IN rule -                                                //
+            // noting that if the lock's null-value behavior is ALLOW, then we actually want IS_NULL_OR_IN, not just IN //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(RecordSecurityLock.NullValueBehavior.ALLOW.equals(recordSecurityLock.getNullValueBehavior()))
+            {
+               lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IS_NULL_OR_IN, securityKeyValues));
+            }
+            else
+            {
+               lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IN, securityKeyValues));
+            }
+         }
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if there's a sourceQueryJoin, then set the lockCriteria on that join - so it gets written into the JOIN ... ON clause //
+      // ... unless we're writing an OR filter.  then we need the condition in the WHERE clause                                //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      boolean doNotPutCriteriaInJoinOn = securityFilterCursor.getBooleanOperator() == QQueryFilter.BooleanOperator.OR;
+      if(sourceQueryJoin != null && !doNotPutCriteriaInJoinOn)
+      {
+         sourceQueryJoin.withSecurityCriteria(lockCriteria);
+      }
+      else
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // we used to add an OR IS NULL for cases of an outer-join - but instead, this is now handled by putting the lockCriteria //
+         // into the join (see above) - so this check is probably deprecated.                                                      //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         /*
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if this field is on the outer side of an outer join, then if we do a straight filter on it, then we're basically      //
+            // nullifying the outer join... so for an outer join use-case, OR the security field criteria with a primary-key IS NULL //
+            // which will make missing rows from the join be found.                                                                  //
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(isOuter)
+            {
+               if(table == null)
+               {
+                  table = QContext.getQInstance().getTable(aliasToTableNameMap.get(tableNameOrAlias));
+               }
+
+               lockFilter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
+               lockFilter.addCriteria(new QFilterCriteria(tableNameOrAlias + "." + table.getPrimaryKeyField(), QCriteriaOperator.IS_BLANK));
+            }
+         */
+
+         /////////////////////////////////////////////////////////////////////////////////////////////////////
+         // If this filter isn't for a queryJoin, then just add it to the main list of security sub-filters //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////
+         this.securityFilterCursor.addSubFilter(lockFilter);
       }
    }
 
@@ -197,9 +628,9 @@ public class JoinsContext
     ** use this method to add to the list, instead of ever adding directly, as it's
     ** important do to that process step (and we've had bugs when it wasn't done).
     *******************************************************************************/
-   private void addQueryJoin(QueryJoin queryJoin, String reason) throws QException
+   private void addQueryJoin(QueryJoin queryJoin, String reason, String logPrefix) throws QException
    {
-      log("Adding query join to context",
+      log(Objects.requireNonNullElse(logPrefix, "") + "Adding query join to context",
          logPair("reason", reason),
          logPair("joinTable", queryJoin.getJoinTable()),
          logPair("joinMetaData.name", () -> queryJoin.getJoinMetaData().getName()),
@@ -208,34 +639,46 @@ public class JoinsContext
       );
       this.queryJoins.add(queryJoin);
       processQueryJoin(queryJoin);
+      dumpDebug(false, false);
    }
 
 
 
    /*******************************************************************************
     ** If there are any joins in the context that don't have a join meta data, see
-    ** if we can find the JoinMetaData to use for them by looking at the main table's
-    ** exposed joins, and using their join paths.
+    ** if we can find the JoinMetaData to use for them by looking at all joins in the
+    ** instance, or at the main table's exposed joins, and using their join paths.
     *******************************************************************************/
-   private void addJoinsFromExposedJoinPaths() throws QException
+   private void fillInMissingJoinMetaData() throws QException
    {
+      log("Begin adding missing join meta data");
+
       ////////////////////////////////////////////////////////////////////////////////
       // do a double-loop, to avoid concurrent modification on the queryJoins list. //
       // that is to say, we'll loop over that list, but possibly add things to it,  //
       // in which case we'll set this flag, and break the inner loop, to go again.  //
       ////////////////////////////////////////////////////////////////////////////////
-      boolean addedJoin;
+      Set<QueryJoin> processedQueryJoins = new HashSet<>();
+      boolean        addedJoin;
       do
       {
          addedJoin = false;
          for(QueryJoin queryJoin : queryJoins)
          {
+            if(processedQueryJoins.contains(queryJoin))
+            {
+               continue;
+            }
+            processedQueryJoins.add(queryJoin);
+
             ///////////////////////////////////////////////////////////////////////////////////////////////
             // if the join has joinMetaData, then we don't need to process it... unless it needs flipped //
             ///////////////////////////////////////////////////////////////////////////////////////////////
             QJoinMetaData joinMetaData = queryJoin.getJoinMetaData();
             if(joinMetaData != null)
             {
+               log("- QueryJoin already has joinMetaData", logPair("joinMetaDataName", joinMetaData.getName()));
+
                boolean isJoinLeftTableInQuery = false;
                String  joinMetaDataLeftTable  = joinMetaData.getLeftTable();
                if(joinMetaDataLeftTable.equals(mainTableName))
@@ -265,7 +708,7 @@ public class JoinsContext
                /////////////////////////////////////////////////////////////////////////////////
                if(!isJoinLeftTableInQuery)
                {
-                  log("Flipping queryJoin because its leftTable wasn't found in the query", logPair("joinMetaDataName", joinMetaData.getName()), logPair("leftTable", joinMetaDataLeftTable));
+                  log("- - Flipping queryJoin because its leftTable wasn't found in the query", logPair("joinMetaDataName", joinMetaData.getName()), logPair("leftTable", joinMetaDataLeftTable));
                   queryJoin.setJoinMetaData(joinMetaData.flip());
                }
             }
@@ -275,11 +718,13 @@ public class JoinsContext
                // try to find a direct join between the main table and this table. //
                // if one is found, then put it (the meta data) on the query join.  //
                //////////////////////////////////////////////////////////////////////
+               log("- QueryJoin doesn't have metaData - looking for it", logPair("joinTableOrItsAlias", queryJoin.getJoinTableOrItsAlias()));
+
                String        baseTableName = Objects.requireNonNullElse(resolveTableNameOrAliasToTableName(queryJoin.getBaseTableOrAlias()), mainTableName);
-               QJoinMetaData found         = findJoinMetaData(instance, baseTableName, queryJoin.getJoinTable());
+               QJoinMetaData found         = findJoinMetaData(baseTableName, queryJoin.getJoinTable(), true);
                if(found != null)
                {
-                  log("Found joinMetaData - setting it in queryJoin", logPair("joinMetaDataName", found.getName()), logPair("baseTableName", baseTableName), logPair("joinTable", queryJoin.getJoinTable()));
+                  log("- - Found joinMetaData - setting it in queryJoin", logPair("joinMetaDataName", found.getName()), logPair("baseTableName", baseTableName), logPair("joinTable", queryJoin.getJoinTable()));
                   queryJoin.setJoinMetaData(found);
                }
                else
@@ -293,7 +738,7 @@ public class JoinsContext
                   {
                      if(queryJoin.getJoinTable().equals(exposedJoin.getJoinTable()))
                      {
-                        log("Found an exposed join", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()), logPair("joinPath", exposedJoin.getJoinPath()));
+                        log("- - Found an exposed join", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()), logPair("joinPath", exposedJoin.getJoinPath()));
 
                         /////////////////////////////////////////////////////////////////////////////////////
                         // loop backward through the join path (from the joinTable back to the main table) //
@@ -304,6 +749,7 @@ public class JoinsContext
                         {
                            String        joinName  = exposedJoin.getJoinPath().get(i);
                            QJoinMetaData joinToAdd = instance.getJoin(joinName);
+                           log("- - - evaluating joinPath element", logPair("i", i), logPair("joinName", joinName));
 
                            /////////////////////////////////////////////////////////////////////////////
                            // get the name from the opposite side of the join (flipping it if needed) //
@@ -332,15 +778,22 @@ public class JoinsContext
                                     queryJoin.setBaseTableOrAlias(nextTable);
                                  }
                                  queryJoin.setJoinMetaData(joinToAdd);
+                                 log("- - - - this is the last element in the join path, so setting this joinMetaData on the original queryJoin");
                               }
                               else
                               {
                                  QueryJoin queryJoinToAdd = makeQueryJoinFromJoinAndTableNames(nextTable, tmpTable, joinToAdd);
                                  queryJoinToAdd.setType(queryJoin.getType());
                                  addedAnyQueryJoins = true;
-                                 this.addQueryJoin(queryJoinToAdd, "forExposedJoin");
+                                 log("- - - - this is not the last element in the join path, so adding a new query join:");
+                                 addQueryJoin(queryJoinToAdd, "forExposedJoin", "- - - - - - ");
+                                 dumpDebug(false, false);
                               }
                            }
+                           else
+                           {
+                              log("- - - - join doesn't need added to the query");
+                           }
 
                            tmpTable = nextTable;
                         }
@@ -361,6 +814,7 @@ public class JoinsContext
       }
       while(addedJoin);
 
+      log("Done adding missing join meta data");
    }
 
 
@@ -370,12 +824,12 @@ public class JoinsContext
     *******************************************************************************/
    private boolean doesJoinNeedAddedToQuery(String joinName)
    {
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // look at all queryJoins already in context - if any have this join's name, then we don't need this join... //
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // look at all queryJoins already in context - if any have this join's name, and aren't implicit-security joins, then we don't need this join... //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       for(QueryJoin queryJoin : queryJoins)
       {
-         if(queryJoin.getJoinMetaData() != null && queryJoin.getJoinMetaData().getName().equals(joinName))
+         if(queryJoin.getJoinMetaData() != null && queryJoin.getJoinMetaData().getName().equals(joinName) && !(queryJoin instanceof ImplicitQueryJoinForSecurityLock))
          {
             return (false);
          }
@@ -395,6 +849,7 @@ public class JoinsContext
       String         tableNameOrAlias = queryJoin.getJoinTableOrItsAlias();
       if(aliasToTableNameMap.containsKey(tableNameOrAlias))
       {
+         dumpDebug(false, true);
          throw (new QException("Duplicate table name or alias: " + tableNameOrAlias));
       }
       aliasToTableNameMap.put(tableNameOrAlias, joinTable.getName());
@@ -439,6 +894,7 @@ public class JoinsContext
          String[] parts = fieldName.split("\\.");
          if(parts.length != 2)
          {
+            dumpDebug(false, true);
             throw new IllegalArgumentException("Mal-formatted field name in query: " + fieldName);
          }
 
@@ -449,6 +905,7 @@ public class JoinsContext
          QTableMetaData table = instance.getTable(tableName);
          if(table == null)
          {
+            dumpDebug(false, true);
             throw new IllegalArgumentException("Could not find table [" + tableName + "] in instance for query");
          }
          return new FieldAndTableNameOrAlias(table.getField(baseFieldName), tableOrAlias);
@@ -503,17 +960,17 @@ public class JoinsContext
 
       for(String filterTable : filterTables)
       {
-         log("Evaluating filterTable", logPair("filterTable", filterTable));
+         log("Evaluating filter", logPair("filterTable", filterTable));
          if(!aliasToTableNameMap.containsKey(filterTable) && !Objects.equals(mainTableName, filterTable))
          {
-            log("- table not in query - adding it", logPair("filterTable", filterTable));
+            log("- table not in query - adding a join for it", logPair("filterTable", filterTable));
             boolean found = false;
             for(QJoinMetaData join : CollectionUtils.nonNullMap(QContext.getQInstance().getJoins()).values())
             {
                QueryJoin queryJoin = makeQueryJoinFromJoinAndTableNames(mainTableName, filterTable, join);
                if(queryJoin != null)
                {
-                  this.addQueryJoin(queryJoin, "forFilter (join found in instance)");
+                  addQueryJoin(queryJoin, "forFilter (join found in instance)", "- - ");
                   found = true;
                   break;
                }
@@ -522,9 +979,13 @@ public class JoinsContext
             if(!found)
             {
                QueryJoin queryJoin = new QueryJoin().withJoinTable(filterTable).withType(QueryJoin.Type.INNER);
-               this.addQueryJoin(queryJoin, "forFilter (join not found in instance)");
+               addQueryJoin(queryJoin, "forFilter (join not found in instance)", "- - ");
             }
          }
+         else
+         {
+            log("- table is already in query - not adding any joins", logPair("filterTable", filterTable));
+         }
       }
    }
 
@@ -566,6 +1027,11 @@ public class JoinsContext
             getTableNameFromFieldNameAndAddToSet(criteria.getOtherFieldName(), filterTables);
          }
 
+         for(QFilterOrderBy orderBy : CollectionUtils.nonNullList(filter.getOrderBys()))
+         {
+            getTableNameFromFieldNameAndAddToSet(orderBy.getFieldName(), filterTables);
+         }
+
          for(QQueryFilter subFilter : CollectionUtils.nonNullList(filter.getSubFilters()))
          {
             populateFilterTablesSet(subFilter, filterTables);
@@ -592,7 +1058,7 @@ public class JoinsContext
    /*******************************************************************************
     **
     *******************************************************************************/
-   public QJoinMetaData findJoinMetaData(QInstance instance, String baseTableName, String joinTableName)
+   public QJoinMetaData findJoinMetaData(String baseTableName, String joinTableName, boolean useExposedJoins)
    {
       List<QJoinMetaData> matches = new ArrayList<>();
       if(baseTableName != null)
@@ -644,7 +1110,29 @@ public class JoinsContext
       }
       else if(matches.size() > 1)
       {
-         throw (new RuntimeException("More than 1 join was found between [" + baseTableName + "] and [" + joinTableName + "].  Specify which one in your QueryJoin."));
+         ////////////////////////////////////////////////////////////////////////////////
+         // if we found more than one join, but we're allowed to useExposedJoins, then //
+         // see if we can tell which match to used based on the table's exposed joins  //
+         ////////////////////////////////////////////////////////////////////////////////
+         if(useExposedJoins)
+         {
+            QTableMetaData mainTable = QContext.getQInstance().getTable(mainTableName);
+            for(ExposedJoin exposedJoin : mainTable.getExposedJoins())
+            {
+               if(exposedJoin.getJoinTable().equals(joinTableName))
+               {
+                  // todo ... is it wrong to always use 0??
+                  return instance.getJoin(exposedJoin.getJoinPath().get(0));
+               }
+            }
+         }
+
+         ///////////////////////////////////////////////
+         // if we couldn't figure it out, then throw. //
+         ///////////////////////////////////////////////
+         dumpDebug(false, true);
+         throw (new RuntimeException("More than 1 join was found between [" + baseTableName + "] and [" + joinTableName + "] "
+            + (useExposedJoins ? "(and exposed joins didn't clarify which one to use). " : "") + "Specify which one in your QueryJoin."));
       }
 
       return (null);
@@ -669,4 +1157,79 @@ public class JoinsContext
       LOG.log(logLevel, message, null, logPairs);
    }
 
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void logFilter(String message, QQueryFilter filter)
+   {
+      if(logLevelForFilter.equals(Level.OFF))
+      {
+         return;
+      }
+      System.out.println(message + "\n" + filter);
+   }
+
+
+
+   /*******************************************************************************
+    ** Print (to stdout, for easier reading) the object in a big table format for
+    ** debugging.  Happens any time logLevel is > OFF.  Not meant for loggly.
+    *******************************************************************************/
+   private void dumpDebug(boolean isStart, boolean isEnd)
+   {
+      if(logLevel.equals(Level.OFF))
+      {
+         return;
+      }
+
+      int sm       = 8;
+      int md       = 30;
+      int lg       = 50;
+      int overhead = 14;
+      int full     = sm + 3 * md + lg + overhead;
+
+      if(isStart)
+      {
+         System.out.println("\n" + StringUtils.safeTruncate("--- Start [main table: " + this.mainTableName + "] " + "-".repeat(full), full));
+      }
+
+      StringBuilder rs           = new StringBuilder();
+      String        formatString = "| %-" + md + "s | %-" + md + "s %-" + md + "s | %-" + lg + "s | %-" + sm + "s |\n";
+      rs.append(String.format(formatString, "Base Table", "Join Table", "(Alias)", "Join Meta Data", "Type"));
+      String dashesLg = "-".repeat(lg);
+      String dashesMd = "-".repeat(md);
+      String dashesSm = "-".repeat(sm);
+      rs.append(String.format(formatString, dashesMd, dashesMd, dashesMd, dashesLg, dashesSm));
+      if(CollectionUtils.nullSafeHasContents(queryJoins))
+      {
+         for(QueryJoin queryJoin : queryJoins)
+         {
+            rs.append(String.format(
+               formatString,
+               StringUtils.hasContent(queryJoin.getBaseTableOrAlias()) ? StringUtils.safeTruncate(queryJoin.getBaseTableOrAlias(), md) : "--",
+               StringUtils.safeTruncate(queryJoin.getJoinTable(), md),
+               (StringUtils.hasContent(queryJoin.getAlias()) ? "(" + StringUtils.safeTruncate(queryJoin.getAlias(), md - 2) + ")" : ""),
+               queryJoin.getJoinMetaData() == null ? "--" : StringUtils.safeTruncate(queryJoin.getJoinMetaData().getName(), lg),
+               queryJoin.getType()));
+         }
+      }
+      else
+      {
+         rs.append(String.format(formatString, "-empty-", "", "", "", ""));
+      }
+
+      System.out.print(rs);
+
+      System.out.println(securityFilter);
+
+      if(isEnd)
+      {
+         System.out.println(StringUtils.safeTruncate("--- End " + "-".repeat(full), full) + "\n");
+      }
+      else
+      {
+         System.out.println(StringUtils.safeTruncate("-".repeat(full), full));
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QCriteriaOperator.java

@@ -49,5 +49,7 @@ public enum QCriteriaOperator
    IS_BLANK,
    IS_NOT_BLANK,
    BETWEEN,
-   NOT_BETWEEN
+   NOT_BETWEEN,
+   TRUE,
+   FALSE
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java

@@ -306,6 +306,11 @@ public class QFilterCriteria implements Serializable, Cloneable
    @Override
    public String toString()
    {
+      if(fieldName == null)
+      {
+         return ("<null-field-criteria>");
+      }
+
       StringBuilder rs = new StringBuilder(fieldName);
       try
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java

@@ -30,6 +30,7 @@ import java.util.Map;
 import java.util.Objects;
 import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
@@ -137,7 +138,7 @@ public class QQueryFilter implements Serializable, Cloneable
 
 
    /*******************************************************************************
-    **
+    ** recursively look at both this filter, and any sub-filters it may have.
     *******************************************************************************/
    public boolean hasAnyCriteria()
    {
@@ -150,7 +151,7 @@ public class QQueryFilter implements Serializable, Cloneable
       {
          for(QQueryFilter subFilter : subFilters)
          {
-            if(subFilter.hasAnyCriteria())
+            if(subFilter != null && subFilter.hasAnyCriteria())
             {
                return (true);
             }
@@ -360,23 +361,44 @@ public class QQueryFilter implements Serializable, Cloneable
       StringBuilder rs = new StringBuilder("(");
       try
       {
+         int criteriaIndex = 0;
          for(QFilterCriteria criterion : CollectionUtils.nonNullList(criteria))
          {
-            rs.append(criterion).append(" ").append(getBooleanOperator()).append(" ");
+            if(criteriaIndex > 0)
+            {
+               rs.append(" ").append(getBooleanOperator()).append(" ");
+            }
+            rs.append(criterion);
+            criteriaIndex++;
          }
 
-         for(QQueryFilter subFilter : CollectionUtils.nonNullList(subFilters))
+         if(CollectionUtils.nullSafeHasContents(subFilters))
          {
-            rs.append(subFilter);
+            rs.append("Sub:{");
+            int subIndex = 0;
+            for(QQueryFilter subFilter : CollectionUtils.nonNullList(subFilters))
+            {
+               if(subIndex > 0)
+               {
+                  rs.append(" ").append(getBooleanOperator()).append(" ");
+               }
+               rs.append(subFilter);
+               subIndex++;
+            }
+            rs.append("}");
          }
+
          rs.append(")");
 
-         rs.append("OrderBy[");
-         for(QFilterOrderBy orderBy : CollectionUtils.nonNullList(orderBys))
+         if(CollectionUtils.nullSafeHasContents(orderBys))
          {
-            rs.append(orderBy).append(",");
+            rs.append("OrderBy[");
+            for(QFilterOrderBy orderBy : CollectionUtils.nonNullList(orderBys))
+            {
+               rs.append(orderBy).append(",");
+            }
+            rs.append("]");
          }
-         rs.append("]");
       }
       catch(Exception e)
       {
@@ -409,9 +431,20 @@ public class QQueryFilter implements Serializable, Cloneable
 
             for(Serializable value : criterion.getValues())
             {
-               String       valueAsString    = ValueUtils.getValueAsString(value);
-               Serializable interpretedValue = variableInterpreter.interpretForObject(valueAsString);
-               newValues.add(interpretedValue);
+               if(value instanceof AbstractFilterExpression<?>)
+               {
+                  /////////////////////////////////////////////////////////////////////////
+                  // todo - do we want to try to interpret values within the expression? //
+                  // e.g., greater than now minus ${input.noOfDays}                      //
+                  /////////////////////////////////////////////////////////////////////////
+                  newValues.add(value);
+               }
+               else
+               {
+                  String       valueAsString    = ValueUtils.getValueAsString(value);
+                  Serializable interpretedValue = variableInterpreter.interpretForObject(valueAsString);
+                  newValues.add(interpretedValue);
+               }
             }
             criterion.setValues(newValues);
          }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryJoin.java

@@ -22,6 +22,8 @@
 package com.kingsrook.qqq.backend.core.model.actions.tables.query;
 
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Objects;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -49,6 +51,10 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
  ** specific joinMetaData to use must be set.  The joinMetaData field can also be
  ** used instead of specify joinTable and baseTableOrAlias, but only for cases
  ** where the baseTable is not an alias.
+ **
+ ** The securityCriteria member, in general, is meant to be populated when a
+ ** JoinsContext is constructed before executing a query, and not meant to be set
+ ** by users.
  *******************************************************************************/
 public class QueryJoin
 {
@@ -59,13 +65,30 @@ public class QueryJoin
    private boolean       select = false;
    private Type          type   = Type.INNER;
 
+   private List<QFilterCriteria> securityCriteria = new ArrayList<>();
+
 
 
    /*******************************************************************************
-    **
+    ** define the types of joins - INNER, LEFT, RIGHT, or FULL.
     *******************************************************************************/
    public enum Type
-   {INNER, LEFT, RIGHT, FULL}
+   {
+      INNER,
+      LEFT,
+      RIGHT,
+      FULL;
+
+
+
+      /*******************************************************************************
+       ** check if a join is an OUTER type (LEFT or RIGHT).
+       *******************************************************************************/
+      public static boolean isOuter(Type type)
+      {
+         return (LEFT == type || RIGHT == type);
+      }
+   }
 
 
 
@@ -348,4 +371,66 @@ public class QueryJoin
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for securityCriteria
+    *******************************************************************************/
+   public List<QFilterCriteria> getSecurityCriteria()
+   {
+      return (this.securityCriteria);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for securityCriteria
+    *******************************************************************************/
+   public void setSecurityCriteria(List<QFilterCriteria> securityCriteria)
+   {
+      this.securityCriteria = securityCriteria;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for securityCriteria
+    *******************************************************************************/
+   public QueryJoin withSecurityCriteria(List<QFilterCriteria> securityCriteria)
+   {
+      this.securityCriteria = securityCriteria;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for securityCriteria
+    *******************************************************************************/
+   public QueryJoin withSecurityCriteria(QFilterCriteria securityCriteria)
+   {
+      if(this.securityCriteria == null)
+      {
+         this.securityCriteria = new ArrayList<>();
+      }
+      this.securityCriteria.add(securityCriteria);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String toString()
+   {
+      return "QueryJoin{base="
+         + baseTableOrAlias + ", joinTable='"
+         + joinTable + ", joinMetaData="
+         + (joinMetaData == null ? null : joinMetaData.getName()) + ", alias='"
+         + alias + ", select="
+         + select + ", type="
+         + type + '}';
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java

@@ -39,7 +39,8 @@ public class ReplaceInput extends AbstractTableActionInput
    private UniqueKey           key;
    private List<QRecord>       records;
    private QQueryFilter        filter;
-   private boolean             performDeletes = true;
+   private boolean             performDeletes            = true;
+   private boolean             allowNullKeyValuesToEqual = false;
 
    private boolean omitDmlAudit = false;
 
@@ -239,4 +240,35 @@ public class ReplaceInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for allowNullKeyValuesToEqual
+    *******************************************************************************/
+   public boolean getAllowNullKeyValuesToEqual()
+   {
+      return (this.allowNullKeyValuesToEqual);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for allowNullKeyValuesToEqual
+    *******************************************************************************/
+   public void setAllowNullKeyValuesToEqual(boolean allowNullKeyValuesToEqual)
+   {
+      this.allowNullKeyValuesToEqual = allowNullKeyValuesToEqual;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for allowNullKeyValuesToEqual
+    *******************************************************************************/
+   public ReplaceInput withAllowNullKeyValuesToEqual(boolean allowNullKeyValuesToEqual)
+   {
+      this.allowNullKeyValuesToEqual = allowNullKeyValuesToEqual;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java

@@ -26,7 +26,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
 
 
 /*******************************************************************************
- **
+ ** Input for Storage actions.
  *******************************************************************************/
 public class StorageInput extends AbstractTableActionInput
 {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/AlertData.java

@@ -0,0 +1,139 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
+
+
+/*******************************************************************************
+ ** Model containing datastructure expected by frontend alert widget
+ **
+ *******************************************************************************/
+public class AlertData extends QWidgetData
+{
+   public enum AlertType
+   {
+      ERROR,
+      SUCCESS,
+      WARNING
+   }
+
+
+
+   private String    html;
+   private AlertType alertType;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public AlertData()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public AlertData(AlertType alertType, String html)
+   {
+      setHtml(html);
+      setAlertType(alertType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for type
+    **
+    *******************************************************************************/
+   public String getType()
+   {
+      return WidgetType.ALERT.getType();
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for html
+    **
+    *******************************************************************************/
+   public String getHtml()
+   {
+      return html;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for html
+    **
+    *******************************************************************************/
+   public void setHtml(String html)
+   {
+      this.html = html;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for html
+    **
+    *******************************************************************************/
+   public AlertData withHtml(String html)
+   {
+      this.html = html;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for alertType
+    *******************************************************************************/
+   public AlertType getAlertType()
+   {
+      return (this.alertType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for alertType
+    *******************************************************************************/
+   public void setAlertType(AlertType alertType)
+   {
+      this.alertType = alertType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for alertType
+    *******************************************************************************/
+   public AlertData withAlertType(AlertType alertType)
+   {
+      this.alertType = alertType;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/FieldValueListData.java

@@ -27,6 +27,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -147,7 +148,7 @@ public class FieldValueListData extends QWidgetData
          }
       }
 
-      QValueFormatter.setDisplayValuesInRecord(fields, record);
+      QValueFormatter.setDisplayValuesInRecord(null, fields.stream().collect(Collectors.toMap(f -> f.getName(), f -> f)), record);
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java

@@ -27,6 +27,7 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
  *******************************************************************************/
 public enum WidgetType
 {
+   ALERT("alert"),
    BAR_CHART("barChart"),
    CHART("chart"),
    CHILD_RECORD_LIST("childRecordList"),
@@ -50,7 +51,9 @@ public enum WidgetType
    USA_MAP("usaMap"),
    COMPOSITE("composite"),
    DATA_BAG_VIEWER("dataBagViewer"),
-   SCRIPT_VIEWER("scriptViewer");
+   SCRIPT_VIEWER("scriptViewer"),
+   REPORT_SETUP("reportSetup"),
+   PIVOT_TABLE_SETUP("pivotTableSetup");
 
 
    private final String type;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java

@@ -27,11 +27,15 @@ import java.lang.reflect.Constructor;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import com.google.common.reflect.ClassPath;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
@@ -43,6 +47,29 @@ public class MetaDataProducerHelper
 {
    private static final QLogger LOG = QLogger.getLogger(MetaDataProducerHelper.class);
 
+   private static Map<Class<?>, Integer> comparatorValuesByType = new HashMap<>();
+   private static Integer                defaultComparatorValue;
+
+   static
+   {
+      ////////////////////////////////////////////////////////////////////////////////////////
+      // define how we break ties in sort-order based on the meta-dta type.  e.g., do apps  //
+      // after all other types (as apps often try to get other types from the instance)     //
+      // also - do backends earlier than others (e.g., tables may expect backends to exist) //
+      // any types not in the map get the default value.                                    //
+      ////////////////////////////////////////////////////////////////////////////////////////
+      comparatorValuesByType.put(QBackendMetaData.class, 1);
+
+      /////////////////////////////////////
+      // unspecified ones will come here //
+      /////////////////////////////////////
+      defaultComparatorValue = 10;
+
+      comparatorValuesByType.put(QJoinMetaData.class, 21);
+      comparatorValuesByType.put(QWidgetMetaData.class, 22);
+      comparatorValuesByType.put(QAppMetaData.class, 23);
+   }
+
 
 
    /*******************************************************************************
@@ -102,11 +129,9 @@ public class MetaDataProducerHelper
          }
       }
 
-      ////////////////////////////////////////////////////////////////////////////////////////////
-      // sort them by sort order, then by the type that they return - specifically - doing apps //
-      // after all other types (as apps often try to get other types from the instance)         //
-      // also - do backends earlier than others (e.g., tables may expect backends to exist)     //
-      ////////////////////////////////////////////////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // sort them by sort order, then by the type that they return, as set up in the static map //
+      /////////////////////////////////////////////////////////////////////////////////////////////
       producers.sort(Comparator
          .comparing((MetaDataProducerInterface<?> p) -> p.getSortOrder())
          .thenComparing((MetaDataProducerInterface<?> p) ->
@@ -114,18 +139,7 @@ public class MetaDataProducerHelper
             try
             {
                Class<?> outputType = p.getClass().getMethod("produce", QInstance.class).getReturnType();
-               if(outputType.equals(QAppMetaData.class))
-               {
-                  return (2);
-               }
-               else if(outputType.equals(QBackendMetaData.class))
-               {
-                  return (0);
-               }
-               else
-               {
-                  return (1);
-               }
+               return comparatorValuesByType.getOrDefault(outputType, defaultComparatorValue);
             }
             catch(Exception e)
             {
@@ -150,7 +164,7 @@ public class MetaDataProducerHelper
             }
             catch(Exception e)
             {
-               LOG.warn("error executing metaDataProducer", logPair("producer", producer.getClass().getSimpleName()), e);
+               LOG.warn("error executing metaDataProducer", e, logPair("producer", producer.getClass().getSimpleName()));
             }
          }
          else

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java

@@ -155,18 +155,6 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
 
 
 
-   /*******************************************************************************
-    ** Fluent setter, returning generically, to help sub-class fluent flows
-    *******************************************************************************/
-   @SuppressWarnings("unchecked")
-   public <T extends QBackendMetaData> T withBackendType(String backendType)
-   {
-      this.backendType = backendType;
-      return (T) this;
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java

@@ -24,11 +24,15 @@ package com.kingsrook.qqq.backend.core.model.metadata.dashboard;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.instances.QInstanceHelpContentManager;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole;
 import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
@@ -61,7 +65,7 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface
 
    protected Map<String, QIcon> icons;
 
-   protected Map<String, QHelpContent> helpContent;
+   protected Map<String, List<QHelpContent>> helpContent;
 
    protected Map<String, Serializable> defaultValues = new LinkedHashMap<>();
 
@@ -691,10 +695,11 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface
    }
 
 
+
    /*******************************************************************************
     ** Getter for helpContent
     *******************************************************************************/
-   public Map<String, QHelpContent> getHelpContent()
+   public Map<String, List<QHelpContent>> getHelpContent()
    {
       return (this.helpContent);
    }
@@ -704,7 +709,7 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface
    /*******************************************************************************
     ** Setter for helpContent
     *******************************************************************************/
-   public void setHelpContent(Map<String, QHelpContent> helpContent)
+   public void setHelpContent(Map<String, List<QHelpContent>> helpContent)
    {
       this.helpContent = helpContent;
    }
@@ -714,11 +719,49 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for helpContent
     *******************************************************************************/
-   public QWidgetMetaData withHelpContent(Map<String, QHelpContent> helpContent)
+   public QWidgetMetaData withHelpContent(Map<String, List<QHelpContent>> helpContent)
    {
       this.helpContent = helpContent;
       return (this);
    }
 
 
+
+   /*******************************************************************************
+    ** Fluent setter for adding 1 helpContent (for a slot)
+    *******************************************************************************/
+   public QWidgetMetaData withHelpContent(String slot, QHelpContent helpContent)
+   {
+      if(this.helpContent == null)
+      {
+         this.helpContent = new HashMap<>();
+      }
+
+      List<QHelpContent> listForSlot = this.helpContent.computeIfAbsent(slot, (k) -> new ArrayList<>());
+      QInstanceHelpContentManager.putHelpContentInList(helpContent, listForSlot);
+
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** remove a helpContent for a slot based on its set of roles
+    *******************************************************************************/
+   public void removeHelpContent(String slot, Set<HelpRole> roles)
+   {
+      if(this.helpContent == null)
+      {
+         return;
+      }
+
+      List<QHelpContent> listForSlot = this.helpContent.get(slot);
+      if(listForSlot == null)
+      {
+         return;
+      }
+
+      QInstanceHelpContentManager.removeHelpContentByRoleSetFromList(roles, listForSlot);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java

@@ -25,10 +25,12 @@ package com.kingsrook.qqq.backend.core.model.metadata.dashboard;
 import java.io.Serializable;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole;
 import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
@@ -235,7 +237,7 @@ public interface QWidgetMetaDataInterface extends MetaDataWithPermissionRules, T
    /*******************************************************************************
     **
     *******************************************************************************/
-   default Map<String, QHelpContent> getHelpContent()
+   default Map<String, List<QHelpContent>> getHelpContent()
    {
       return (null);
    }
@@ -244,11 +246,29 @@ public interface QWidgetMetaDataInterface extends MetaDataWithPermissionRules, T
    /*******************************************************************************
     **
     *******************************************************************************/
-   default void setHelpContent(Map<String, QHelpContent> helpContent)
+   default void setHelpContent(Map<String, List<QHelpContent>> helpContent)
    {
       LOG.debug("Setting help content in a widgetMetaData type that doesn't support it (because it didn't override the getter/setter)");
    }
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default QWidgetMetaDataInterface withHelpContent(String slot, QHelpContent helpContent)
+   {
+      LOG.debug("Setting help content in a widgetMetaData type that doesn't support it (because it didn't override the getter/setter)");
+      return (this);
+   }
+
+   /*******************************************************************************
+    ** remove a helpContent for a slot based on its set of roles
+    *******************************************************************************/
+   default void removeHelpContent(String slot, Set<HelpRole> roles)
+   {
+      LOG.debug("Setting help content in a widgetMetaData type that doesn't support it (because it didn't override the getter/setter)");
+   }
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownData.java

@@ -28,6 +28,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.dashboard;
  *******************************************************************************/
 public class WidgetDropdownData
 {
+   private String  name;
    private String  possibleValueSourceName;
    private String  foreignKeyFieldName;
    private String  label;
@@ -44,6 +45,9 @@ public class WidgetDropdownData
    ////////////////////////////////////////////////////////////////////////////////////////////////
    private String labelForNullValue;
 
+   private WidgetDropdownType type = WidgetDropdownType.POSSIBLE_VALUE_SOURCE;
+
+
 
    /*******************************************************************************
     ** Getter for possibleValueSourceName
@@ -366,4 +370,65 @@ public class WidgetDropdownData
    }
 
 
+
+   /*******************************************************************************
+    ** Getter for type
+    *******************************************************************************/
+   public WidgetDropdownType getType()
+   {
+      return (this.type);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for type
+    *******************************************************************************/
+   public void setType(WidgetDropdownType type)
+   {
+      this.type = type;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for type
+    *******************************************************************************/
+   public WidgetDropdownData withType(WidgetDropdownType type)
+   {
+      this.type = type;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for name
+    *******************************************************************************/
+   public String getName()
+   {
+      return (this.name);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for name
+    *******************************************************************************/
+   public void setName(String name)
+   {
+      this.name = name;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for name
+    *******************************************************************************/
+   public WidgetDropdownData withName(String name)
+   {
+      this.name = name;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownType.java

@@ -0,0 +1,33 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.dashboard;
+
+
+/*******************************************************************************
+ ** Possible types for widget dropdowns
+ **
+ *******************************************************************************/
+public enum WidgetDropdownType
+{
+   POSSIBLE_VALUE_SOURCE,
+   DATE_PICKER
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehavior.java

@@ -0,0 +1,359 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.fields;
+
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Field Display Behavior class for customizing the display values used
+ ** in date-time fields
+ *******************************************************************************/
+public class DateTimeDisplayValueBehavior implements FieldDisplayBehavior<DateTimeDisplayValueBehavior>
+{
+   private static final QLogger LOG = QLogger.getLogger(DateTimeDisplayValueBehavior.class);
+
+   private String zoneIdFromFieldName;
+   private String fallbackZoneId;
+
+   private String defaultZoneId;
+
+   private static DateTimeDisplayValueBehavior NOOP = new DateTimeDisplayValueBehavior();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public DateTimeDisplayValueBehavior getDefault()
+   {
+      return NOOP;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
+   {
+      if(StringUtils.hasContent(defaultZoneId))
+      {
+         applyDefaultZoneId(recordList, table, field);
+      }
+      else if(StringUtils.hasContent(zoneIdFromFieldName))
+      {
+         applyZoneIdFromFieldName(recordList, table, field);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void applyDefaultZoneId(List<QRecord> recordList, QTableMetaData table, QFieldMetaData field)
+   {
+      for(QRecord record : CollectionUtils.nonNullList(recordList))
+      {
+         try
+         {
+            Instant instant = record.getValueInstant(field.getName());
+
+            if(instant == null)
+            {
+               continue;
+            }
+
+            ZonedDateTime zonedDateTime = instant.atZone(ZoneId.of(defaultZoneId));
+            record.setDisplayValue(field.getName(), QValueFormatter.formatDateTimeWithZone(zonedDateTime));
+         }
+         catch(Exception e)
+         {
+            LOG.info("Error applying defaultZoneId DateTimeDisplayValueBehavior", logPair("table", table.getName()), logPair("field", field.getName()), logPair("id", record.getValue(table.getPrimaryKeyField())));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void applyZoneIdFromFieldName(List<QRecord> recordList, QTableMetaData table, QFieldMetaData field)
+   {
+      for(QRecord record : CollectionUtils.nonNullList(recordList))
+      {
+         try
+         {
+            Instant instant = record.getValueInstant(field.getName());
+            if(instant == null)
+            {
+               continue;
+            }
+
+            String zoneString = record.getValueString(zoneIdFromFieldName);
+
+            ZoneId zoneId = null;
+            if(StringUtils.hasContent(zoneString))
+            {
+               try
+               {
+                  zoneId = ZoneId.of(zoneString);
+               }
+               catch(Exception e)
+               {
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // we probably(?) don't need a stack trace here (and it could get noisy?), so just info w/ the exception message... //
+                  // and we expect this might be somewhat frequent, if you might have invalid values in your zoneId field...          //
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  LOG.info("Exception applying zoneIdFromFieldName behavior", logPair("message", e.getMessage()), logPair("table", table.getName()), logPair("field", field.getName()), logPair("id", record.getValue(table.getPrimaryKeyField())));
+               }
+            }
+
+            if(zoneId == null)
+            {
+               ////////////////////////////////////////////////////////////////////////////////////////////////
+               // if the zone string from the other field isn't valid, and we have a fallback, try to use it //
+               ////////////////////////////////////////////////////////////////////////////////////////////////
+               if(StringUtils.hasContent(fallbackZoneId))
+               {
+                  ////////////////////////////////////////////////////////////////////////////////////////////
+                  // assume that validation has confirmed this is a valid zone - so no try-catch right here //
+                  ////////////////////////////////////////////////////////////////////////////////////////////
+                  zoneId = ZoneId.of(fallbackZoneId);
+               }
+            }
+
+            if(zoneId != null)
+            {
+               ZonedDateTime zonedDateTime = instant.atZone(zoneId);
+               record.setDisplayValue(field.getName(), QValueFormatter.formatDateTimeWithZone(zonedDateTime));
+            }
+         }
+         catch(Exception e)
+         {
+            ///////////////////////////////////////////////////////////////////////
+            // we don't expect this to ever hit - so warn it w/ stack if it does //
+            ///////////////////////////////////////////////////////////////////////
+            LOG.warn("Unexpected error applying zoneIdFromFieldName behavior", e, logPair("table", table.getName()), logPair("field", field.getName()), logPair("id", record.getValue(table.getPrimaryKeyField())));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<String> validateBehaviorConfiguration(QTableMetaData tableMetaData, QFieldMetaData fieldMetaData)
+   {
+      List<String> errors      = new ArrayList<>();
+      String       errorSuffix = " field [" + fieldMetaData.getName() + "] in table [" + tableMetaData.getName() + "]";
+
+      if(!QFieldType.DATE_TIME.equals(fieldMetaData.getType()))
+      {
+         errors.add("A DateTimeDisplayValueBehavior was a applied to a non-DATE_TIME" + errorSuffix);
+      }
+
+      //////////////////////////////////////////////////
+      // validate rules if zoneIdFromFieldName is set //
+      //////////////////////////////////////////////////
+      if(StringUtils.hasContent(zoneIdFromFieldName))
+      {
+         if(StringUtils.hasContent(defaultZoneId))
+         {
+            errors.add("You may not specify both zoneIdFromFieldName and defaultZoneId in DateTimeDisplayValueBehavior on" + errorSuffix);
+         }
+
+         if(!tableMetaData.getFields().containsKey(zoneIdFromFieldName))
+         {
+            errors.add("Unrecognized field name [" + zoneIdFromFieldName + "] for [zoneIdFromFieldName] in DateTimeDisplayValueBehavior on" + errorSuffix);
+         }
+         else
+         {
+            QFieldMetaData zoneIdField = tableMetaData.getFields().get(zoneIdFromFieldName);
+            if(!QFieldType.STRING.equals(zoneIdField.getType()))
+            {
+               errors.add("A non-STRING type [" + zoneIdField.getType() + "] was specified as the zoneIdFromFieldName field [" + zoneIdFromFieldName + "] in DateTimeDisplayValueBehavior on" + errorSuffix);
+            }
+         }
+      }
+
+      ////////////////////////////////////////////
+      // validate rules if defaultZoneId is set //
+      ////////////////////////////////////////////
+      if(StringUtils.hasContent(defaultZoneId))
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         // would check that you didn't specify from zoneIdFromFieldName - but that's covered above //
+         /////////////////////////////////////////////////////////////////////////////////////////////
+
+         if(StringUtils.hasContent(fallbackZoneId))
+         {
+            errors.add("You may not specify both defaultZoneId and fallbackZoneId in DateTimeDisplayValueBehavior on" + errorSuffix);
+         }
+
+         try
+         {
+            ZoneId.of(defaultZoneId);
+         }
+         catch(Exception e)
+         {
+            errors.add("Invalid ZoneId [" + defaultZoneId + "] for [defaultZoneId] in DateTimeDisplayValueBehavior on" + errorSuffix + "; " + e.getMessage());
+         }
+      }
+
+      /////////////////////////////////////////////
+      // validate rules if fallbackZoneId is set //
+      /////////////////////////////////////////////
+      if(StringUtils.hasContent(fallbackZoneId))
+      {
+         if(!StringUtils.hasContent(zoneIdFromFieldName))
+         {
+            errors.add("You may only set fallbackZoneId if using zoneIdFromFieldName in DateTimeDisplayValueBehavior on" + errorSuffix);
+         }
+
+         try
+         {
+            ZoneId.of(fallbackZoneId);
+         }
+         catch(Exception e)
+         {
+            errors.add("Invalid ZoneId [" + fallbackZoneId + "] for [fallbackZoneId] in DateTimeDisplayValueBehavior on" + errorSuffix + "; " + e.getMessage());
+         }
+      }
+
+      return (errors);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for zoneIdFromFieldName
+    *******************************************************************************/
+   public String getZoneIdFromFieldName()
+   {
+      return (this.zoneIdFromFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for zoneIdFromFieldName
+    *******************************************************************************/
+   public void setZoneIdFromFieldName(String zoneIdFromFieldName)
+   {
+      this.zoneIdFromFieldName = zoneIdFromFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for zoneIdFromFieldName
+    *******************************************************************************/
+   public DateTimeDisplayValueBehavior withZoneIdFromFieldName(String zoneIdFromFieldName)
+   {
+      this.zoneIdFromFieldName = zoneIdFromFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for defaultZoneId
+    *******************************************************************************/
+   public String getDefaultZoneId()
+   {
+      return (this.defaultZoneId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for defaultZoneId
+    *******************************************************************************/
+   public void setDefaultZoneId(String defaultZoneId)
+   {
+      this.defaultZoneId = defaultZoneId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for defaultZoneId
+    *******************************************************************************/
+   public DateTimeDisplayValueBehavior withDefaultZoneId(String defaultZoneId)
+   {
+      this.defaultZoneId = defaultZoneId;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for fallbackZoneId
+    *******************************************************************************/
+   public String getFallbackZoneId()
+   {
+      return (this.fallbackZoneId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fallbackZoneId
+    *******************************************************************************/
+   public void setFallbackZoneId(String fallbackZoneId)
+   {
+      this.fallbackZoneId = fallbackZoneId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fallbackZoneId
+    *******************************************************************************/
+   public DateTimeDisplayValueBehavior withFallbackZoneId(String fallbackZoneId)
+   {
+      this.fallbackZoneId = fallbackZoneId;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehavior.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -26,7 +26,6 @@ import java.io.Serializable;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.util.List;
-import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -70,16 +69,12 @@ public enum DynamicDefaultValueBehavior implements FieldBehavior<DynamicDefaultV
     **
     *******************************************************************************/
    @Override
-   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field, Set<FieldBehavior<?>> behaviorsToOmit)
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
    {
       if(this.equals(NONE))
       {
          return;
       }
-      if(behaviorsToOmit != null && behaviorsToOmit.contains(this))
-      {
-         return;
-      }
 
       switch(this)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldBehavior.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -22,8 +22,9 @@
 package com.kingsrook.qqq.backend.core.model.metadata.fields;
 
 
+import java.util.Collections;
 import java.util.List;
-import java.util.Set;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -34,8 +35,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
  ** Interface for (expected to be?) enums which define behaviors that get applied
  ** to fields.
  **
- ** At the present, these behaviors get applied before a field is stored (insert
- ** or update), through the ValueBehaviorApplier class.
+ ** Some of these behaviors get applied before a field is stored (insert
+ ** or update), through the ValueBehaviorApplier class.  Others can be used to
+ ** do more advanced display formatting than the displayFormat string alone can
+ ** do (see QValueFormatter).
  **
  *******************************************************************************/
 public interface FieldBehavior<T extends FieldBehavior<T>>
@@ -45,12 +48,13 @@ public interface FieldBehavior<T extends FieldBehavior<T>>
     ** In case a behavior of this type wasn't set on the field, what should the
     ** default of this type be?
     *******************************************************************************/
+   @JsonIgnore
    T getDefault();
 
    /*******************************************************************************
     ** Apply this behavior to a list of records
     *******************************************************************************/
-   void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field, Set<FieldBehavior<?>> behaviorsToOmit);
+   void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field);
 
    /*******************************************************************************
     ** control if multiple behaviors of this type should be allowed together on a field.
@@ -60,4 +64,14 @@ public interface FieldBehavior<T extends FieldBehavior<T>>
       return (false);
    }
 
+   /*******************************************************************************
+    ** allow this behavior to be validated during QInstance validation.
+    **
+    ** return a list of validation errors, if there are any.
+    *******************************************************************************/
+   default List<String> validateBehaviorConfiguration(QTableMetaData tableMetaData, QFieldMetaData fieldMetaData)
+   {
+      return (Collections.emptyList());
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldDisplayBehavior.java

@@ -0,0 +1,31 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.fields;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public interface FieldDisplayBehavior<T extends FieldDisplayBehavior<T>> extends FieldBehavior<T>
+{
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java

@@ -721,6 +721,17 @@ public class QFieldMetaData implements Cloneable
       {
          return (behaviorType.getEnumConstants()[0].getDefault());
       }
+      else
+      {
+         try
+         {
+            return (behaviorType.getConstructor().newInstance().getDefault());
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Error getting default behaviorType for [" + behaviorType.getSimpleName() + "]", e);
+         }
+      }
 
       return (null);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/ValueTooLongBehavior.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields;
 
 
 import java.util.List;
-import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -66,16 +65,12 @@ public enum ValueTooLongBehavior implements FieldBehavior<ValueTooLongBehavior>
     **
     *******************************************************************************/
    @Override
-   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field, Set<FieldBehavior<?>> behaviorsToOmit)
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
    {
       if(this.equals(PASS_THROUGH))
       {
          return;
       }
-      if(behaviorsToOmit != null && behaviorsToOmit.contains(this))
-      {
-         return;
-      }
 
       String fieldName = field.getName();
       if(!QFieldType.STRING.equals(field.getType()))

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java

@@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
@@ -75,6 +76,8 @@ public class QFrontendTableMetaData
    private boolean usesVariants;
    private String  variantTableLabel;
 
+   private ShareableTableMetaData shareableTableMetaData;
+
    //////////////////////////////////////////////////////////////////////////////////
    // do not add setters.  take values from the source-object in the constructor!! //
    //////////////////////////////////////////////////////////////////////////////////
@@ -104,6 +107,8 @@ public class QFrontendTableMetaData
          }
 
          this.sections = tableMetaData.getSections();
+
+         this.shareableTableMetaData = tableMetaData.getShareableTableMetaData();
       }
 
       if(includeJoins)
@@ -367,4 +372,14 @@ public class QFrontendTableMetaData
       return (this.variantTableLabel);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for shareableTableMetaData
+    **
+    *******************************************************************************/
+   public ShareableTableMetaData getShareableTableMetaData()
+   {
+      return shareableTableMetaData;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java

@@ -59,9 +59,9 @@ public class QFrontendWidgetMetaData
    private boolean showReloadButton = false;
    private boolean showExportButton = false;
 
-   protected Map<String, QIcon>        icons;
-   protected Map<String, QHelpContent> helpContent;
-   protected Map<String, Serializable> defaultValues;
+   protected Map<String, QIcon>              icons;
+   protected Map<String, List<QHelpContent>> helpContent;
+   protected Map<String, Serializable>       defaultValues;
 
    private final boolean hasPermission;
 
@@ -273,7 +273,7 @@ public class QFrontendWidgetMetaData
     ** Getter for helpContent
     **
     *******************************************************************************/
-   public Map<String, QHelpContent> getHelpContent()
+   public Map<String, List<QHelpContent>> getHelpContent()
    {
       return helpContent;
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java

@@ -53,6 +53,7 @@ public class QPossibleValueSource implements TopLevelMetaDataInterface
    // for type = TABLE //
    //////////////////////
    private String               tableName;
+   private String overrideIdField;
    private List<String>         searchFields;
    private List<QFilterOrderBy> orderByFields;
 
@@ -630,4 +631,35 @@ public class QPossibleValueSource implements TopLevelMetaDataInterface
       qInstance.addPossibleValueSource(this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for overrideIdField
+    *******************************************************************************/
+   public String getOverrideIdField()
+   {
+      return (this.overrideIdField);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for overrideIdField
+    *******************************************************************************/
+   public void setOverrideIdField(String overrideIdField)
+   {
+      this.overrideIdField = overrideIdField;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for overrideIdField
+    *******************************************************************************/
+   public QPossibleValueSource withOverrideIdField(String overrideIdField)
+   {
+      this.overrideIdField = overrideIdField;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/MultiRecordSecurityLock.java

@@ -0,0 +1,198 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.security;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** Subclass of RecordSecurityLock, for combining multiple locks using a boolean
+ ** (AND/OR) condition.  Note that the combined locks can themselves also be
+ ** Multi-locks, thus creating a tree of locks.
+ *******************************************************************************/
+public class MultiRecordSecurityLock extends RecordSecurityLock implements Cloneable
+{
+   private List<RecordSecurityLock> locks = new ArrayList<>();
+   private BooleanOperator          operator;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   protected MultiRecordSecurityLock clone() throws CloneNotSupportedException
+   {
+      MultiRecordSecurityLock clone = (MultiRecordSecurityLock) super.clone();
+
+      /////////////////////////
+      // deep-clone the list //
+      /////////////////////////
+      if(locks != null)
+      {
+         clone.locks = new ArrayList<>();
+         for(RecordSecurityLock lock : locks)
+         {
+            clone.locks.add(lock.clone());
+         }
+      }
+
+      return (clone);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public enum BooleanOperator
+   {
+      AND,
+      OR;
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public QQueryFilter.BooleanOperator toFilterOperator()
+      {
+         return switch(this)
+         {
+            case AND -> QQueryFilter.BooleanOperator.AND;
+            case OR -> QQueryFilter.BooleanOperator.OR;
+         };
+      }
+   }
+
+
+
+   ////////////////////////////////
+   // todo - remove, this is POC //
+   ////////////////////////////////
+   static
+   {
+      new QTableMetaData()
+         .withName("savedReport")
+         .withRecordSecurityLock(new MultiRecordSecurityLock()
+            .withLocks(List.of(
+               new RecordSecurityLock()
+                  .withFieldName("userId")
+                  .withSecurityKeyType("user")
+                  .withNullValueBehavior(NullValueBehavior.DENY)
+                  .withLockScope(LockScope.READ_AND_WRITE),
+               new RecordSecurityLock()
+                  .withFieldName("sharedReport.userId")
+                  .withJoinNameChain(List.of("reportJoinSharedReport"))
+                  .withSecurityKeyType("user")
+                  .withNullValueBehavior(NullValueBehavior.DENY)
+                  .withLockScope(LockScope.READ_AND_WRITE), // dynamic, from a value...
+               new RecordSecurityLock()
+                  .withFieldName("sharedReport.groupId")
+                  .withJoinNameChain(List.of("reportJoinSharedReport"))
+                  .withSecurityKeyType("group")
+                  .withNullValueBehavior(NullValueBehavior.DENY)
+                  .withLockScope(LockScope.READ_AND_WRITE) // dynamic, from a value...
+            )));
+
+   }
+
+   /*******************************************************************************
+    ** Getter for locks
+    *******************************************************************************/
+   public List<RecordSecurityLock> getLocks()
+   {
+      return (this.locks);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for locks
+    *******************************************************************************/
+   public void setLocks(List<RecordSecurityLock> locks)
+   {
+      this.locks = locks;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for locks
+    *******************************************************************************/
+   public MultiRecordSecurityLock withLocks(List<RecordSecurityLock> locks)
+   {
+      this.locks = locks;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluently add one lock
+    *******************************************************************************/
+   public MultiRecordSecurityLock withLock(RecordSecurityLock lock)
+   {
+      if(this.locks == null)
+      {
+         this.locks = new ArrayList<>();
+      }
+      this.locks.add(lock);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for operator
+    *******************************************************************************/
+   public BooleanOperator getOperator()
+   {
+      return (this.operator);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for operator
+    *******************************************************************************/
+   public void setOperator(BooleanOperator operator)
+   {
+      this.operator = operator;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for operator
+    *******************************************************************************/
+   public MultiRecordSecurityLock withOperator(BooleanOperator operator)
+   {
+      this.operator = operator;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLock.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.model.metadata.security;
 
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -41,7 +42,7 @@ import java.util.Map;
  ** - READ_AND_WRITE means that users cannot read or write records without a valid key.
  ** - WRITE means that users cannot write records without a valid key (but they can read them).
  *******************************************************************************/
-public class RecordSecurityLock
+public class RecordSecurityLock implements Cloneable
 {
    private String            securityKeyType;
    private String            fieldName;
@@ -52,6 +53,28 @@ public class RecordSecurityLock
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   protected RecordSecurityLock clone() throws CloneNotSupportedException
+   {
+      RecordSecurityLock clone = (RecordSecurityLock) super.clone();
+
+      /////////////////////////
+      // deep-clone the list //
+      /////////////////////////
+      if(joinNameChain != null)
+      {
+         clone.joinNameChain = new ArrayList<>();
+         clone.joinNameChain.addAll(joinNameChain);
+      }
+
+      return (clone);
+   }
+
+
+
    /*******************************************************************************
     ** Constructor
     **
@@ -106,8 +129,9 @@ public class RecordSecurityLock
     *******************************************************************************/
    public enum LockScope
    {
-      READ_AND_WRITE,
-      WRITE
+      READ_AND_WRITE, // lock both reads and writes
+      WRITE, // only lock writes
+      READ // only lock reads
    }
 
 
@@ -265,4 +289,22 @@ public class RecordSecurityLock
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String toString()
+   {
+      return "RecordSecurityLock{"
+         + "securityKeyType='" + securityKeyType + '\''
+         + ", fieldName='" + fieldName + '\''
+         + ", joinNameChain=" + joinNameChain
+         + ", nullValueBehavior=" + nullValueBehavior
+         + ", lockScope=" + lockScope
+         + '}';
+   }
+
 }
+

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLockFilters.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.security;
 
 
 import java.util.List;
+import java.util.Set;
 
 
 /*******************************************************************************
@@ -46,6 +47,65 @@ public class RecordSecurityLockFilters
 
 
 
+   /*******************************************************************************
+    ** filter a list of locks so that we only see the ones that apply to reads.
+    *******************************************************************************/
+   public static MultiRecordSecurityLock filterForReadLockTree(List<RecordSecurityLock> recordSecurityLocks)
+   {
+      return filterForLockTree(recordSecurityLocks, Set.of(RecordSecurityLock.LockScope.READ_AND_WRITE, RecordSecurityLock.LockScope.READ));
+   }
+
+
+
+   /*******************************************************************************
+    ** filter a list of locks so that we only see the ones that apply to writes.
+    *******************************************************************************/
+   public static MultiRecordSecurityLock filterForWriteLockTree(List<RecordSecurityLock> recordSecurityLocks)
+   {
+      return filterForLockTree(recordSecurityLocks, Set.of(RecordSecurityLock.LockScope.READ_AND_WRITE, RecordSecurityLock.LockScope.WRITE));
+   }
+
+
+
+   /*******************************************************************************
+    ** filter a list of locks so that we only see the ones that apply to any of the
+    ** input set of scopes.
+    *******************************************************************************/
+   private static MultiRecordSecurityLock filterForLockTree(List<RecordSecurityLock> recordSecurityLocks, Set<RecordSecurityLock.LockScope> allowedScopes)
+   {
+      if(recordSecurityLocks == null)
+      {
+         return (null);
+      }
+
+      //////////////////////////////////////////////////////////////
+      // at the top-level we build a multi-lock with AND operator //
+      //////////////////////////////////////////////////////////////
+      MultiRecordSecurityLock result = new MultiRecordSecurityLock();
+      result.setOperator(MultiRecordSecurityLock.BooleanOperator.AND);
+
+      for(RecordSecurityLock recordSecurityLock : recordSecurityLocks)
+      {
+         if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock)
+         {
+            MultiRecordSecurityLock filteredSubLock = filterForLockTree(multiRecordSecurityLock.getLocks(), allowedScopes);
+            filteredSubLock.setOperator(multiRecordSecurityLock.getOperator());
+            result.withLock(filteredSubLock);
+         }
+         else
+         {
+            if(allowedScopes.contains(recordSecurityLock.getLockScope()))
+            {
+               result.withLock(recordSecurityLock);
+            }
+         }
+      }
+
+      return (result);
+   }
+
+
+
    /*******************************************************************************
     ** filter a list of locks so that we only see the ones that apply to writes.
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/sharing/ShareScopePossibleValueMetaDataProducer.java

@@ -0,0 +1,48 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.sharing;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.processes.implementations.sharing.ShareScope;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class ShareScopePossibleValueMetaDataProducer implements MetaDataProducerInterface<QPossibleValueSource>
+{
+   public static final String NAME = "shareScope";
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QPossibleValueSource produce(QInstance qInstance) throws QException
+   {
+      return QPossibleValueSource.newForEnum(NAME, ShareScope.values());
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/sharing/ShareableAudienceType.java

@@ -0,0 +1,186 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.sharing;
+
+
+import java.io.Serializable;
+
+
+/*******************************************************************************
+ ** As a component of a ShareableTableMetaData instance, define details about
+ ** one particular audience type.
+ **
+ ** e.g., if a table can be shared to users and groups, there'd be 2 instances of
+ ** this object - one like:
+ ** - name: user
+ ** - fieldName: userId
+ ** - sourceTableName: User.TABLE_NAME
+ ** - sourceTableKeyFieldName: email (e.g., can be a UK, not just the PKey)
+ **
+ ** and another similar, w/ the group-type details.
+ *******************************************************************************/
+public class ShareableAudienceType implements Serializable
+{
+   private String name;
+   private String fieldName;
+   private String sourceTableName;
+
+   /////////////////////////////////////////////////////////////////////////////////////////////////////
+   // maybe normally the primary key in the source table, but could be a unique-key instead sometimes //
+   /////////////////////////////////////////////////////////////////////////////////////////////////////
+   private String sourceTableKeyFieldName;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public ShareableAudienceType()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for name
+    *******************************************************************************/
+   public String getName()
+   {
+      return (this.name);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for name
+    *******************************************************************************/
+   public void setName(String name)
+   {
+      this.name = name;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for name
+    *******************************************************************************/
+   public ShareableAudienceType withName(String name)
+   {
+      this.name = name;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for fieldName
+    *******************************************************************************/
+   public String getFieldName()
+   {
+      return (this.fieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldName
+    *******************************************************************************/
+   public void setFieldName(String fieldName)
+   {
+      this.fieldName = fieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldName
+    *******************************************************************************/
+   public ShareableAudienceType withFieldName(String fieldName)
+   {
+      this.fieldName = fieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for sourceTableName
+    *******************************************************************************/
+   public String getSourceTableName()
+   {
+      return (this.sourceTableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sourceTableName
+    *******************************************************************************/
+   public void setSourceTableName(String sourceTableName)
+   {
+      this.sourceTableName = sourceTableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for sourceTableName
+    *******************************************************************************/
+   public ShareableAudienceType withSourceTableName(String sourceTableName)
+   {
+      this.sourceTableName = sourceTableName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for sourceTableKeyFieldName
+    *******************************************************************************/
+   public String getSourceTableKeyFieldName()
+   {
+      return (this.sourceTableKeyFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sourceTableKeyFieldName
+    *******************************************************************************/
+   public void setSourceTableKeyFieldName(String sourceTableKeyFieldName)
+   {
+      this.sourceTableKeyFieldName = sourceTableKeyFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for sourceTableKeyFieldName
+    *******************************************************************************/
+   public ShareableAudienceType withSourceTableKeyFieldName(String sourceTableKeyFieldName)
+   {
+      this.sourceTableKeyFieldName = sourceTableKeyFieldName;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/sharing/ShareableTableMetaData.java

@@ -0,0 +1,398 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.sharing;
+
+
+import java.io.Serializable;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** meta data to attach to a table, to describe that its records are shareable.
+ *******************************************************************************/
+public class ShareableTableMetaData implements Serializable
+{
+   ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   // this is the name of the table that is a many-to-one join to the table whose records are being shared.  //
+   // not the table whose records are shared (the asset table)                                               //
+   // for example:  given that we want to share "savedReports", the value here could be "sharedSavedReports" //
+   // and this object will be attached to the savedReports table.                                            //
+   ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   private String sharedRecordTableName;
+
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////
+   // name of the field in the sharedRecordTable that has a foreign key pointing at the asset table //
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////
+   private String assetIdFieldName;
+
+   //////////////////////////////////////////////////////
+   // name of the scope field in the sharedRecordTable //
+   //////////////////////////////////////////////////////
+   private String scopeFieldName;
+
+   ///////////////////////////////////////////////////////////
+   // map of audienceTypes names to type definition objects //
+   ///////////////////////////////////////////////////////////
+   private Map<String, ShareableAudienceType> audienceTypes;
+
+   /////////////////////////////////////////////////
+   // PVS that lists the available audience types //
+   /////////////////////////////////////////////////
+   private String audienceTypesPossibleValueSourceName;
+
+   ///////////////////////////////////////////////////
+   // PVS that lists the available audience records //
+   ///////////////////////////////////////////////////
+   private String audiencePossibleValueSourceName;
+
+   //////////////////////////////////////////////////////////////
+   // name of a field in "this" table, that has the owner's id //
+   //////////////////////////////////////////////////////////////
+   private String thisTableOwnerIdFieldName;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public ShareableTableMetaData()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for sharedRecordTableName
+    *******************************************************************************/
+   public String getSharedRecordTableName()
+   {
+      return (this.sharedRecordTableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sharedRecordTableName
+    *******************************************************************************/
+   public void setSharedRecordTableName(String sharedRecordTableName)
+   {
+      this.sharedRecordTableName = sharedRecordTableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for sharedRecordTableName
+    *******************************************************************************/
+   public ShareableTableMetaData withSharedRecordTableName(String sharedRecordTableName)
+   {
+      this.sharedRecordTableName = sharedRecordTableName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for assetIdFieldName
+    *******************************************************************************/
+   public String getAssetIdFieldName()
+   {
+      return (this.assetIdFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for assetIdFieldName
+    *******************************************************************************/
+   public void setAssetIdFieldName(String assetIdFieldName)
+   {
+      this.assetIdFieldName = assetIdFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for assetIdFieldName
+    *******************************************************************************/
+   public ShareableTableMetaData withAssetIdFieldName(String assetIdFieldName)
+   {
+      this.assetIdFieldName = assetIdFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for scopeFieldName
+    *******************************************************************************/
+   public String getScopeFieldName()
+   {
+      return (this.scopeFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for scopeFieldName
+    *******************************************************************************/
+   public void setScopeFieldName(String scopeFieldName)
+   {
+      this.scopeFieldName = scopeFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for scopeFieldName
+    *******************************************************************************/
+   public ShareableTableMetaData withScopeFieldName(String scopeFieldName)
+   {
+      this.scopeFieldName = scopeFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for audienceTypes
+    *******************************************************************************/
+   public Map<String, ShareableAudienceType> getAudienceTypes()
+   {
+      return (this.audienceTypes);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for audienceTypes
+    *******************************************************************************/
+   public void setAudienceTypes(Map<String, ShareableAudienceType> audienceTypes)
+   {
+      this.audienceTypes = audienceTypes;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for audienceTypes
+    *******************************************************************************/
+   public ShareableTableMetaData withAudienceTypes(Map<String, ShareableAudienceType> audienceTypes)
+   {
+      this.audienceTypes = audienceTypes;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for audienceTypes
+    *******************************************************************************/
+   public ShareableTableMetaData withAudienceType(ShareableAudienceType audienceType)
+   {
+      if(this.audienceTypes == null)
+      {
+         this.audienceTypes = new LinkedHashMap<>();
+      }
+
+      if(audienceType.getName() == null)
+      {
+         throw (new IllegalArgumentException("Attempt to add an audience type without a name"));
+      }
+
+      if(this.audienceTypes.containsKey(audienceType.getName()))
+      {
+         throw (new IllegalArgumentException("Attempt to add more than 1 audience type with the same name [" + audienceType.getName() + "]"));
+      }
+
+      this.audienceTypes.put(audienceType.getName(), audienceType);
+
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for audienceTypesPossibleValueSourceName
+    *******************************************************************************/
+   public String getAudienceTypesPossibleValueSourceName()
+   {
+      return (this.audienceTypesPossibleValueSourceName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for audienceTypesPossibleValueSourceName
+    *******************************************************************************/
+   public void setAudienceTypesPossibleValueSourceName(String audienceTypesPossibleValueSourceName)
+   {
+      this.audienceTypesPossibleValueSourceName = audienceTypesPossibleValueSourceName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for audienceTypesPossibleValueSourceName
+    *******************************************************************************/
+   public ShareableTableMetaData withAudienceTypesPossibleValueSourceName(String audienceTypesPossibleValueSourceName)
+   {
+      this.audienceTypesPossibleValueSourceName = audienceTypesPossibleValueSourceName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for thisTableOwnerIdFieldName
+    *******************************************************************************/
+   public String getThisTableOwnerIdFieldName()
+   {
+      return (this.thisTableOwnerIdFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for thisTableOwnerIdFieldName
+    *******************************************************************************/
+   public void setThisTableOwnerIdFieldName(String thisTableOwnerIdFieldName)
+   {
+      this.thisTableOwnerIdFieldName = thisTableOwnerIdFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for thisTableOwnerIdFieldName
+    *******************************************************************************/
+   public ShareableTableMetaData withThisTableOwnerIdFieldName(String thisTableOwnerIdFieldName)
+   {
+      this.thisTableOwnerIdFieldName = thisTableOwnerIdFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for audiencePossibleValueSourceName
+    *******************************************************************************/
+   public String getAudiencePossibleValueSourceName()
+   {
+      return (this.audiencePossibleValueSourceName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for audiencePossibleValueSourceName
+    *******************************************************************************/
+   public void setAudiencePossibleValueSourceName(String audiencePossibleValueSourceName)
+   {
+      this.audiencePossibleValueSourceName = audiencePossibleValueSourceName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for audiencePossibleValueSourceName
+    *******************************************************************************/
+   public ShareableTableMetaData withAudiencePossibleValueSourceName(String audiencePossibleValueSourceName)
+   {
+      this.audiencePossibleValueSourceName = audiencePossibleValueSourceName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void validate(QInstance qInstance, QTableMetaData tableMetaData, QInstanceValidator qInstanceValidator)
+   {
+      String prefix = "ShareableTableMetaData for table [" + tableMetaData.getName() + "]: ";
+      if(qInstanceValidator.assertCondition(StringUtils.hasContent(sharedRecordTableName), prefix + "missing sharedRecordTableName."))
+      {
+         boolean hasAssetIdFieldName = qInstanceValidator.assertCondition(StringUtils.hasContent(assetIdFieldName), prefix + "missing assetIdFieldName");
+         boolean hasScopeFieldName   = qInstanceValidator.assertCondition(StringUtils.hasContent(scopeFieldName), prefix + "missing scopeFieldName");
+
+         QTableMetaData sharedRecordTable         = qInstance.getTable(sharedRecordTableName);
+         boolean        hasValidSharedRecordTable = qInstanceValidator.assertCondition(sharedRecordTable != null, prefix + "unrecognized sharedRecordTableName [" + sharedRecordTableName + "]");
+
+         if(hasValidSharedRecordTable && hasAssetIdFieldName)
+         {
+            qInstanceValidator.assertCondition(sharedRecordTable.getFields().containsKey(assetIdFieldName), prefix + "unrecognized assertIdFieldName [" + assetIdFieldName + "] in sharedRecordTable [" + sharedRecordTableName + "]");
+         }
+
+         if(hasValidSharedRecordTable && hasScopeFieldName)
+         {
+            qInstanceValidator.assertCondition(sharedRecordTable.getFields().containsKey(scopeFieldName), prefix + "unrecognized scopeFieldName [" + scopeFieldName + "] in sharedRecordTable [" + sharedRecordTableName + "]");
+         }
+
+         if(qInstanceValidator.assertCondition(CollectionUtils.nullSafeHasContents(audienceTypes), prefix + "missing audienceTypes"))
+         {
+            for(Map.Entry<String, ShareableAudienceType> entry : audienceTypes.entrySet())
+            {
+               ShareableAudienceType audienceType = entry.getValue();
+               qInstanceValidator.assertCondition(Objects.equals(entry.getKey(), audienceType.getName()), prefix + "inconsistent naming for shareableAudienceType [" + entry.getKey() + "] != [" + audienceType.getName() + "]");
+               if(qInstanceValidator.assertCondition(StringUtils.hasContent(audienceType.getFieldName()), prefix + "missing fieldName for shareableAudienceType [" + entry.getKey() + "]") && hasValidSharedRecordTable)
+               {
+                  qInstanceValidator.assertCondition(sharedRecordTable.getFields().containsKey(audienceType.getFieldName()), prefix + "unrecognized fieldName [" + audienceType.getFieldName() + "] for shareableAudienceType [" + entry.getKey() + "] in sharedRecordTable [" + sharedRecordTableName + "]");
+               }
+
+               // todo - validate this audienceType.getSourceTableKeyFieldName() is a field, and it is a UKey
+
+               /* todo - make these optional i guess, because i didn't put user table in qqq
+               boolean hasSourceTableKeyFieldName = qInstanceValidator.assertCondition(StringUtils.hasContent(audienceType.getSourceTableKeyFieldName()), prefix + "missing sourceTableKeyFieldName for shareableAudienceType [" + entry.getKey() + "]");
+               if(qInstanceValidator.assertCondition(qInstance.getTable(audienceType.getSourceTableName()) != null, prefix + "unrecognized sourceTableName [" + audienceType.getSourceTableName() + "] for shareableAudienceType [" + entry.getKey() + "] in sharedRecordTable [" + sharedRecordTableName + "]") && hasSourceTableKeyFieldName)
+               {
+                  qInstanceValidator.assertCondition(qInstance.getTable(audienceType.getSourceTableName()).getFields().containsKey(audienceType.getSourceTableKeyFieldName()), prefix + "unrecognized sourceTableKeyFieldName [" + audienceType.getSourceTableKeyFieldName() + "] for shareableAudienceType [" + entry.getKey() + "] in sharedRecordTable [" + sharedRecordTableName + "]");
+               }
+               */
+            }
+         }
+      }
+
+      if(StringUtils.hasContent(thisTableOwnerIdFieldName))
+      {
+         qInstanceValidator.assertCondition(tableMetaData.getFields().containsKey(thisTableOwnerIdFieldName), prefix + "unrecognized thisTableOwnerIdFieldName [" + thisTableOwnerIdFieldName + "]");
+      }
+
+      if(StringUtils.hasContent(audienceTypesPossibleValueSourceName))
+      {
+         qInstanceValidator.assertCondition(qInstance.getPossibleValueSource(audienceTypesPossibleValueSourceName) != null, prefix + "unrecognized audienceTypesPossibleValueSourceName [" + audienceTypesPossibleValueSourceName + "]");
+      }
+
+      if(StringUtils.hasContent(audiencePossibleValueSourceName))
+      {
+         qInstanceValidator.assertCondition(qInstance.getPossibleValueSource(audiencePossibleValueSourceName) != null, prefix + "unrecognized audiencePossibleValueSourceName [" + audiencePossibleValueSourceName + "]");
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java

@@ -48,6 +48,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -107,6 +108,7 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData
 
    private List<ExposedJoin> exposedJoins;
 
+   private ShareableTableMetaData shareableTableMetaData;
 
 
    /*******************************************************************************
@@ -1385,4 +1387,35 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData
       return (this);
    }
 
+
+   /*******************************************************************************
+    ** Getter for shareableTableMetaData
+    *******************************************************************************/
+   public ShareableTableMetaData getShareableTableMetaData()
+   {
+      return (this.shareableTableMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for shareableTableMetaData
+    *******************************************************************************/
+   public void setShareableTableMetaData(ShareableTableMetaData shareableTableMetaData)
+   {
+      this.shareableTableMetaData = shareableTableMetaData;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for shareableTableMetaData
+    *******************************************************************************/
+   public QTableMetaData withShareableTableMetaData(ShareableTableMetaData shareableTableMetaData)
+   {
+      this.shareableTableMetaData = shareableTableMetaData;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportColumns.java

@@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.model.savedreports;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
@@ -35,6 +37,7 @@ public class ReportColumns implements Serializable
    private List<ReportColumn> columns;
 
 
+
    /*******************************************************************************
     ** Getter for columns
     *******************************************************************************/
@@ -45,6 +48,23 @@ public class ReportColumns implements Serializable
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public List<ReportColumn> extractVisibleColumns()
+   {
+      return CollectionUtils.nonNullList(getColumns()).stream()
+         //////////////////////////////////////////////////////
+         // if isVisible is missing, we assume it to be true //
+         //////////////////////////////////////////////////////
+         .filter(rc -> rc.getIsVisible() == null || rc.getIsVisible())
+         .filter(rc -> StringUtils.hasContent(rc.getName()))
+         .filter(rc -> !rc.getName().startsWith("__check"))
+         .toList();
+   }
+
+
+
    /*******************************************************************************
     ** Setter for columns
     *******************************************************************************/
@@ -65,6 +85,7 @@ public class ReportColumns implements Serializable
    }
 
 
+
    /*******************************************************************************
     ** Fluent setter to add 1 column
     *******************************************************************************/
@@ -79,6 +100,7 @@ public class ReportColumns implements Serializable
    }
 
 
+
    /*******************************************************************************
     ** Fluent setter to add 1 column w/ just a name
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReport.java

@@ -48,13 +48,13 @@ public class SavedReport extends QRecordEntity
    @QField(isEditable = false)
    private Instant modifyDate;
 
-   @QField(isRequired = true, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS)
+   @QField(isRequired = true, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS, label = "Report Name")
    private String label;
 
-   @QField(possibleValueSourceName = TablesPossibleValueSourceMetaDataProvider.NAME, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR)
-   private String tableName; // todo - qqqTableId... ?
+   @QField(possibleValueSourceName = TablesPossibleValueSourceMetaDataProvider.NAME, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, label = "Table", isRequired = true)
+   private String tableName;
 
-   @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, dynamicDefaultValueBehavior = DynamicDefaultValueBehavior.USER_ID)
+   @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, dynamicDefaultValueBehavior = DynamicDefaultValueBehavior.USER_ID, label = "Owner")
    private String userId;
 
    @QField(label = "Query Filter")

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportJsonFieldDisplayValueFormatter.java

@@ -0,0 +1,150 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.savedreports;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.SavedReportToReportMetaDataAdapter;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class SavedReportJsonFieldDisplayValueFormatter implements FieldDisplayBehavior<SavedReportJsonFieldDisplayValueFormatter>
+{
+   private static SavedReportJsonFieldDisplayValueFormatter savedReportJsonFieldDisplayValueFormatter = null;
+
+
+
+   /*******************************************************************************
+    ** Singleton constructor
+    *******************************************************************************/
+   private SavedReportJsonFieldDisplayValueFormatter()
+   {
+
+   }
+
+
+
+   /*******************************************************************************
+    ** Singleton accessor
+    *******************************************************************************/
+   public static SavedReportJsonFieldDisplayValueFormatter getInstance()
+   {
+      if(savedReportJsonFieldDisplayValueFormatter == null)
+      {
+         savedReportJsonFieldDisplayValueFormatter = new SavedReportJsonFieldDisplayValueFormatter();
+      }
+      return (savedReportJsonFieldDisplayValueFormatter);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public SavedReportJsonFieldDisplayValueFormatter getDefault()
+   {
+      return getInstance();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
+   {
+      for(QRecord record : CollectionUtils.nonNullList(recordList))
+      {
+         if(field.getName().equals("queryFilterJson"))
+         {
+            String queryFilterJson = record.getValueString("queryFilterJson");
+            if(StringUtils.hasContent(queryFilterJson))
+            {
+               try
+               {
+                  QQueryFilter qQueryFilter  = SavedReportToReportMetaDataAdapter.getQQueryFilter(queryFilterJson);
+                  int          criteriaCount = CollectionUtils.nonNullList(qQueryFilter.getCriteria()).size();
+                  record.setDisplayValue("queryFilterJson", criteriaCount + " Filter" + StringUtils.plural(criteriaCount));
+               }
+               catch(Exception e)
+               {
+                  record.setDisplayValue("queryFilterJson", "Invalid Filter...");
+               }
+            }
+         }
+
+         if(field.getName().equals("columnsJson"))
+         {
+            String columnsJson = record.getValueString("columnsJson");
+            if(StringUtils.hasContent(columnsJson))
+            {
+               try
+               {
+                  ReportColumns reportColumns = SavedReportToReportMetaDataAdapter.getReportColumns(columnsJson);
+                  int           columnCount   = reportColumns.extractVisibleColumns().size();
+
+                  record.setDisplayValue("columnsJson", columnCount + " Column" + StringUtils.plural(columnCount));
+               }
+               catch(Exception e)
+               {
+                  record.setDisplayValue("columnsJson", "Invalid Columns...");
+               }
+            }
+         }
+
+         if(field.getName().equals("pivotTableJson"))
+         {
+            String pivotTableJson = record.getValueString("pivotTableJson");
+            if(StringUtils.hasContent(pivotTableJson))
+            {
+               try
+               {
+                  PivotTableDefinition pivotTableDefinition = SavedReportToReportMetaDataAdapter.getPivotTableDefinition(pivotTableJson);
+                  int                  rowCount             = CollectionUtils.nonNullList(pivotTableDefinition.getRows()).size();
+                  int                  columnCount          = CollectionUtils.nonNullList(pivotTableDefinition.getColumns()).size();
+                  int                  valueCount           = CollectionUtils.nonNullList(pivotTableDefinition.getValues()).size();
+                  record.setDisplayValue("pivotTableJson", rowCount + " Row" + StringUtils.plural(rowCount) + ", " + columnCount + " Column" + StringUtils.plural(columnCount) + ", and " + valueCount + " Value" + StringUtils.plural(valueCount));
+               }
+               catch(Exception e)
+               {
+                  record.setDisplayValue("pivotTableJson", "Invalid Pivot Table...");
+               }
+            }
+         }
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizer.java

@@ -0,0 +1,282 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.savedreports;
+
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
+import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.SavedReportToReportMetaDataAdapter;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class SavedReportTableCustomizer implements TableCustomizerInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      return (preInsertOrUpdate(records));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      return (preInsertOrUpdate(records));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<QRecord> preInsertOrUpdate(List<QRecord> records)
+   {
+      for(QRecord record : CollectionUtils.nonNullList(records))
+      {
+         preValidateRecord(record);
+      }
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   void preValidateRecord(QRecord record)
+   {
+      try
+      {
+         String tableName       = record.getValueString("tableName");
+         String queryFilterJson = record.getValueString("queryFilterJson");
+         String columnsJson     = record.getValueString("columnsJson");
+         String pivotTableJson  = record.getValueString("pivotTableJson");
+
+         Set<String> usedColumns = new HashSet<>();
+
+         QTableMetaData table = QContext.getQInstance().getTable(tableName);
+         if(table == null)
+         {
+            record.addError(new BadInputStatusMessage("Unrecognized table name: " + tableName));
+         }
+
+         if(StringUtils.hasContent(queryFilterJson))
+         {
+            try
+            {
+               ////////////////////////////////////////////////////////////////
+               // nothing to validate on filter, other than, we can parse it //
+               ////////////////////////////////////////////////////////////////
+               SavedReportToReportMetaDataAdapter.getQQueryFilter(queryFilterJson);
+            }
+            catch(IOException e)
+            {
+               record.addError(new BadInputStatusMessage("Unable to parse queryFilterJson: " + e.getMessage()));
+            }
+         }
+
+         boolean hadColumnParseError = false;
+         if(StringUtils.hasContent(columnsJson))
+         {
+            try
+            {
+               /////////////////////////////////////////////////////////////////////////
+               // make sure we can parse columns, and that we have at least 1 visible //
+               /////////////////////////////////////////////////////////////////////////
+               ReportColumns reportColumns = SavedReportToReportMetaDataAdapter.getReportColumns(columnsJson);
+               for(ReportColumn column : reportColumns.extractVisibleColumns())
+               {
+                  usedColumns.add(column.getName());
+               }
+            }
+            catch(IOException e)
+            {
+               record.addError(new BadInputStatusMessage("Unable to parse columnsJson: " + e.getMessage()));
+               hadColumnParseError = true;
+            }
+         }
+
+         if(usedColumns.isEmpty() && !hadColumnParseError)
+         {
+            record.addError(new BadInputStatusMessage("A Report must contain at least 1 column"));
+         }
+
+         if(StringUtils.hasContent(pivotTableJson))
+         {
+            try
+            {
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // make sure we can parse pivot table, and we have ... at least 1 ... row?  maybe that's all that's needed //
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               PivotTableDefinition pivotTableDefinition          = SavedReportToReportMetaDataAdapter.getPivotTableDefinition(pivotTableJson);
+               boolean              anyRows                       = false;
+               boolean              missingAnyFieldNamesInRows    = false;
+               boolean              missingAnyFieldNamesInColumns = false;
+               boolean              missingAnyFieldNamesInValues  = false;
+               boolean              missingAnyFunctionsInValues   = false;
+
+               //////////////////
+               // look at rows //
+               //////////////////
+               for(PivotTableGroupBy row : CollectionUtils.nonNullList(pivotTableDefinition.getRows()))
+               {
+                  anyRows = true;
+                  if(StringUtils.hasContent(row.getFieldName()))
+                  {
+                     if(!usedColumns.contains(row.getFieldName()) && !hadColumnParseError)
+                     {
+                        record.addError(new BadInputStatusMessage("A pivot table row is using field (" + getFieldLabelElseName(table, row.getFieldName()) + ") which is not an active column on this report."));
+                     }
+                  }
+                  else
+                  {
+                     missingAnyFieldNamesInRows = true;
+                  }
+               }
+
+               if(!anyRows)
+               {
+                  record.addError(new BadInputStatusMessage("A Pivot Table must contain at least 1 row"));
+               }
+
+               /////////////////////
+               // look at columns //
+               /////////////////////
+               for(PivotTableGroupBy column : CollectionUtils.nonNullList(pivotTableDefinition.getColumns()))
+               {
+                  if(StringUtils.hasContent(column.getFieldName()))
+                  {
+                     if(!usedColumns.contains(column.getFieldName()) && !hadColumnParseError)
+                     {
+                        record.addError(new BadInputStatusMessage("A pivot table column is using field (" + getFieldLabelElseName(table, column.getFieldName()) + ") which is not an active column on this report."));
+                     }
+                  }
+                  else
+                  {
+                     missingAnyFieldNamesInColumns = true;
+                  }
+               }
+
+               ////////////////////
+               // look at values //
+               ////////////////////
+               for(PivotTableValue value : CollectionUtils.nonNullList(pivotTableDefinition.getValues()))
+               {
+                  if(StringUtils.hasContent(value.getFieldName()))
+                  {
+                     if(!usedColumns.contains(value.getFieldName()) && !hadColumnParseError)
+                     {
+                        record.addError(new BadInputStatusMessage("A pivot table value is using field (" + getFieldLabelElseName(table, value.getFieldName()) + ") which is not an active column on this report."));
+                     }
+                  }
+                  else
+                  {
+                     missingAnyFieldNamesInValues = true;
+                  }
+
+                  if(value.getFunction() == null)
+                  {
+                     missingAnyFunctionsInValues = true;
+                  }
+               }
+
+               ////////////////////////////////////////////////
+               // errors based on missing things found above //
+               ////////////////////////////////////////////////
+               if(missingAnyFieldNamesInRows)
+               {
+                  record.addError(new BadInputStatusMessage("Missing field name for at least one pivot table row."));
+               }
+
+               if(missingAnyFieldNamesInColumns)
+               {
+                  record.addError(new BadInputStatusMessage("Missing field name for at least one pivot table column."));
+               }
+
+               if(missingAnyFieldNamesInValues)
+               {
+                  record.addError(new BadInputStatusMessage("Missing field name for at least one pivot table value."));
+               }
+
+               if(missingAnyFunctionsInValues)
+               {
+                  record.addError(new BadInputStatusMessage("Missing function for at least one pivot table value."));
+               }
+            }
+            catch(IOException e)
+            {
+               record.addError(new BadInputStatusMessage("Unable to parse pivotTableJson: " + e.getMessage()));
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error validating a savedReport");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String getFieldLabelElseName(QTableMetaData table, String fieldName)
+   {
+      try
+      {
+         GenerateReportAction.FieldAndJoinTable fieldAndJoinTable = GenerateReportAction.getFieldAndJoinTable(table, fieldName);
+         return (fieldAndJoinTable.getLabel(table));
+      }
+      catch(Exception e)
+      {
+         return (fieldName);
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java

@@ -24,20 +24,35 @@ package com.kingsrook.qqq.backend.core.model.savedreports;
 
 import java.util.List;
 import java.util.function.Consumer;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.DefaultWidgetRenderer;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableAudienceType;
+import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.RenderSavedReportMetaDataProducer;
 
 
@@ -48,6 +63,7 @@ public class SavedReportsMetaDataProvider
 {
    public static final String REPORT_STORAGE_TABLE_NAME = "reportStorage";
 
+   public static final String SHARED_SAVED_REPORT_JOIN_SAVED_REPORT = "sharedSavedReportJoinSavedReport";
 
 
    /*******************************************************************************
@@ -70,7 +86,33 @@ public class SavedReportsMetaDataProvider
          .findFirst()
          .ifPresent(f -> f.setDefaultValue(REPORT_STORAGE_TABLE_NAME));
 
-      // todo - when we build the UI instance.addWidget(defineReportSetupWidget());
+      instance.addWidget(defineReportSetupWidget());
+      instance.addWidget(definePivotTableSetupWidget());
+
+      /////////////////////////////////////
+      // todo - param to enable sharing? //
+      /////////////////////////////////////
+      instance.addTable(defineSharedSavedReportTable(recordTablesBackendName, backendDetailEnricher));
+      instance.addJoin(defineSharedSavedReportJoinSavedReport());
+      if(instance.getPossibleValueSource(ShareScopePossibleValueMetaDataProducer.NAME) == null)
+      {
+         instance.addPossibleValueSource(new ShareScopePossibleValueMetaDataProducer().produce(new QInstance()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QJoinMetaData defineSharedSavedReportJoinSavedReport()
+   {
+      return (new QJoinMetaData()
+         .withName(SHARED_SAVED_REPORT_JOIN_SAVED_REPORT)
+         .withLeftTable(SharedSavedReport.TABLE_NAME)
+         .withRightTable(SavedReport.TABLE_NAME)
+         .withType(JoinType.MANY_TO_ONE)
+         .withJoinOn(new JoinOn("savedReportId", "id")));
    }
 
 
@@ -95,20 +137,35 @@ public class SavedReportsMetaDataProvider
       return (table);
    }
 
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
-   /* todo - when we build the UI
    private QWidgetMetaDataInterface defineReportSetupWidget()
    {
       return new QWidgetMetaData()
          .withName("reportSetupWidget")
-         .withLabel("Report Setup")
+         .withLabel("Filters and Columns")
          .withIsCard(true)
          .withType(WidgetType.REPORT_SETUP.getType())
          .withCodeReference(new QCodeReference(DefaultWidgetRenderer.class));
    }
-   */
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QWidgetMetaDataInterface definePivotTableSetupWidget()
+   {
+      return new QWidgetMetaData()
+         .withName("pivotTableSetupWidget")
+         .withLabel("Pivot Table")
+         .withIsCard(true)
+         .withType(WidgetType.PIVOT_TABLE_SETUP.getType())
+         .withCodeReference(new QCodeReference(DefaultWidgetRenderer.class));
+   }
 
 
 
@@ -119,24 +176,66 @@ public class SavedReportsMetaDataProvider
    {
       QTableMetaData table = new QTableMetaData()
          .withName(SavedReport.TABLE_NAME)
+         .withLabel("Report")
          .withIcon(new QIcon().withName("article"))
          .withRecordLabelFormat("%s")
          .withRecordLabelFields("label")
          .withBackendName(backendName)
          .withPrimaryKeyField("id")
          .withFieldsFromEntity(SavedReport.class)
-         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label")))
-         .withSection(new QFieldSection("settings", new QIcon().withName("settings"), Tier.T2, List.of("tableName")))
-         // todo - turn on when building UI .withSection(new QFieldSection("reportSetup", new QIcon().withName("table_chart"), Tier.T2).withWidgetName("reportSetupWidget"))
-         .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("queryFilterJson", "columnsJson", "pivotTableJson")))
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "tableName")))
+         .withSection(new QFieldSection("filtersAndColumns", new QIcon().withName("table_chart"), Tier.T2).withLabel("Filters and Columns").withWidgetName("reportSetupWidget"))
+         .withSection(new QFieldSection("pivotTable", new QIcon().withName("pivot_table_chart"), Tier.T2).withLabel("Pivot Table").withWidgetName("pivotTableSetupWidget"))
+         .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("queryFilterJson", "columnsJson", "pivotTableJson")).withIsHidden(true))
          .withSection(new QFieldSection("hidden", new QIcon().withName("text_snippet"), Tier.T2, List.of("inputFieldsJson", "userId")).withIsHidden(true))
          .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
 
-      for(String jsonFieldName : List.of("queryFilterJson", "columnsJson", "inputFieldsJson", "pivotTableJson"))
+      table.getField("queryFilterJson").withBehavior(SavedReportJsonFieldDisplayValueFormatter.getInstance());
+      table.getField("columnsJson").withBehavior(SavedReportJsonFieldDisplayValueFormatter.getInstance());
+      table.getField("pivotTableJson").withBehavior(SavedReportJsonFieldDisplayValueFormatter.getInstance());
+
+      table.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(SavedReportTableCustomizer.class));
+      table.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(SavedReportTableCustomizer.class));
+
+      table.withShareableTableMetaData(new ShareableTableMetaData()
+         .withSharedRecordTableName(SharedSavedReport.TABLE_NAME)
+         .withAssetIdFieldName("savedReportId")
+         .withScopeFieldName("scope")
+         .withThisTableOwnerIdFieldName("userId")
+         .withAudienceType(new ShareableAudienceType().withName("user").withFieldName("userId")));
+
+      if(backendDetailEnricher != null)
       {
-         table.getField(jsonFieldName).withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
+         backendDetailEnricher.accept(table);
       }
 
+      return (table);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QTableMetaData defineSharedSavedReportTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   {
+      QTableMetaData table = new QTableMetaData()
+         .withName(SharedSavedReport.TABLE_NAME)
+         .withLabel("Shared Report")
+         .withIcon(new QIcon().withName("share"))
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("savedReportId")
+         .withBackendName(backendName)
+         .withUniqueKey(new UniqueKey("savedReportId", "userId"))
+         .withPrimaryKeyField("id")
+         .withFieldsFromEntity(SharedSavedReport.class)
+         // todo - security key
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "savedReportId", "userId")))
+         .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("scope")))
+         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
+
       if(backendDetailEnricher != null)
       {
          backendDetailEnricher.accept(table);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SharedSavedReport.java

@@ -0,0 +1,267 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.savedreports;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer;
+
+
+/*******************************************************************************
+ ** Entity bean for the shared saved report table
+ *******************************************************************************/
+public class SharedSavedReport extends QRecordEntity
+{
+   public static final String TABLE_NAME = "sharedSavedReport";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField(isEditable = false)
+   private Instant createDate;
+
+   @QField(isEditable = false)
+   private Instant modifyDate;
+
+   @QField(possibleValueSourceName = SavedReport.TABLE_NAME, label = "Report")
+   private Integer savedReportId;
+
+   @QField(label = "User")
+   private String userId;
+
+   @QField(possibleValueSourceName = ShareScopePossibleValueMetaDataProducer.NAME)
+   private String scope;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public SharedSavedReport()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public SharedSavedReport(QRecord qRecord) throws QException
+   {
+      populateFromQRecord(qRecord);
+   }
+
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return (this.id);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    *******************************************************************************/
+   public SharedSavedReport withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for createDate
+    *******************************************************************************/
+   public Instant getCreateDate()
+   {
+      return (this.createDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for createDate
+    *******************************************************************************/
+   public void setCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for createDate
+    *******************************************************************************/
+   public SharedSavedReport withCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for modifyDate
+    *******************************************************************************/
+   public Instant getModifyDate()
+   {
+      return (this.modifyDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for modifyDate
+    *******************************************************************************/
+   public void setModifyDate(Instant modifyDate)
+   {
+      this.modifyDate = modifyDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for modifyDate
+    *******************************************************************************/
+   public SharedSavedReport withModifyDate(Instant modifyDate)
+   {
+      this.modifyDate = modifyDate;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for savedReportId
+    *******************************************************************************/
+   public Integer getSavedReportId()
+   {
+      return (this.savedReportId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for savedReportId
+    *******************************************************************************/
+   public void setSavedReportId(Integer savedReportId)
+   {
+      this.savedReportId = savedReportId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for savedReportId
+    *******************************************************************************/
+   public SharedSavedReport withSavedReportId(Integer savedReportId)
+   {
+      this.savedReportId = savedReportId;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for userId
+    *******************************************************************************/
+   public String getUserId()
+   {
+      return (this.userId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for userId
+    *******************************************************************************/
+   public void setUserId(String userId)
+   {
+      this.userId = userId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for userId
+    *******************************************************************************/
+   public SharedSavedReport withUserId(String userId)
+   {
+      this.userId = userId;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for scope
+    *******************************************************************************/
+   public String getScope()
+   {
+      return (this.scope);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for scope
+    *******************************************************************************/
+   public void setScope(String scope)
+   {
+      this.scope = scope;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for scope
+    *******************************************************************************/
+   public SharedSavedReport withScope(String scope)
+   {
+      this.scope = scope;
+      return (this);
+   }
+
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SavedViewsMetaDataProvider.java

@@ -26,15 +26,22 @@ import java.util.List;
 import java.util.function.Consumer;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
+import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.processes.implementations.savedviews.DeleteSavedViewProcess;
 import com.kingsrook.qqq.backend.core.processes.implementations.savedviews.QuerySavedViewProcess;
 import com.kingsrook.qqq.backend.core.processes.implementations.savedviews.StoreSavedViewProcess;
@@ -45,6 +52,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.savedviews.Store
  *******************************************************************************/
 public class SavedViewsMetaDataProvider
 {
+   public static final String SHARED_SAVED_VIEW_JOIN_SAVED_VIEW = "sharedSavedViewJoinSavedView";
 
 
    /*******************************************************************************
@@ -57,6 +65,16 @@ public class SavedViewsMetaDataProvider
       instance.addProcess(QuerySavedViewProcess.getProcessMetaData());
       instance.addProcess(StoreSavedViewProcess.getProcessMetaData());
       instance.addProcess(DeleteSavedViewProcess.getProcessMetaData());
+
+      /////////////////////////////////////
+      // todo - param to enable sharing? //
+      /////////////////////////////////////
+      instance.addTable(defineSharedSavedViewTable(backendName, backendDetailEnricher));
+      instance.addJoin(defineSharedSavedViewJoinSavedView());
+      if(instance.getPossibleValueSource(ShareScopePossibleValueMetaDataProducer.NAME) == null)
+      {
+         instance.addPossibleValueSource(new ShareScopePossibleValueMetaDataProducer().produce(new QInstance()));
+      }
    }
 
 
@@ -68,7 +86,7 @@ public class SavedViewsMetaDataProvider
    {
       QTableMetaData table = new QTableMetaData()
          .withName(SavedView.TABLE_NAME)
-         .withLabel("Saved View")
+         .withLabel("View")
          .withIcon(new QIcon().withName("table_view"))
          .withRecordLabelFormat("%s")
          .withRecordLabelFields("label")
@@ -104,4 +122,50 @@ public class SavedViewsMetaDataProvider
          .withOrderByField("label");
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QTableMetaData defineSharedSavedViewTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   {
+      QTableMetaData table = new QTableMetaData()
+         .withName(SharedSavedView.TABLE_NAME)
+         .withLabel("Shared View")
+         .withIcon(new QIcon().withName("share"))
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("savedViewId")
+         .withBackendName(backendName)
+         .withUniqueKey(new UniqueKey("savedViewId", "userId"))
+         .withPrimaryKeyField("id")
+         .withFieldsFromEntity(SharedSavedView.class)
+         // todo - security key
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "savedViewId", "userId")))
+         .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("scope")))
+         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
+
+      if(backendDetailEnricher != null)
+      {
+         backendDetailEnricher.accept(table);
+      }
+
+      return (table);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QJoinMetaData defineSharedSavedViewJoinSavedView()
+   {
+      return (new QJoinMetaData()
+         .withName(SHARED_SAVED_VIEW_JOIN_SAVED_VIEW)
+         .withLeftTable(SharedSavedView.TABLE_NAME)
+         .withRightTable(SavedView.TABLE_NAME)
+         .withType(JoinType.MANY_TO_ONE)
+         .withJoinOn(new JoinOn("savedViewId", "id")));
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SharedSavedView.java

@@ -0,0 +1,265 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.savedviews;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer;
+
+
+/*******************************************************************************
+ ** Entity bean for the shared saved view table
+ *******************************************************************************/
+public class SharedSavedView extends QRecordEntity
+{
+   public static final String TABLE_NAME = "sharedSavedView";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField(isEditable = false)
+   private Instant createDate;
+
+   @QField(isEditable = false)
+   private Instant modifyDate;
+
+   @QField(possibleValueSourceName = SavedView.TABLE_NAME, label = "View")
+   private Integer savedViewId;
+
+   @QField(label = "User")
+   private String userId;
+
+   @QField(possibleValueSourceName = ShareScopePossibleValueMetaDataProducer.NAME)
+   private String scope;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public SharedSavedView()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public SharedSavedView(QRecord qRecord) throws QException
+   {
+      populateFromQRecord(qRecord);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return (this.id);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    *******************************************************************************/
+   public com.kingsrook.qqq.backend.core.model.savedviews.SharedSavedView withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for createDate
+    *******************************************************************************/
+   public Instant getCreateDate()
+   {
+      return (this.createDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for createDate
+    *******************************************************************************/
+   public void setCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for createDate
+    *******************************************************************************/
+   public com.kingsrook.qqq.backend.core.model.savedviews.SharedSavedView withCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for modifyDate
+    *******************************************************************************/
+   public Instant getModifyDate()
+   {
+      return (this.modifyDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for modifyDate
+    *******************************************************************************/
+   public void setModifyDate(Instant modifyDate)
+   {
+      this.modifyDate = modifyDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for modifyDate
+    *******************************************************************************/
+   public com.kingsrook.qqq.backend.core.model.savedviews.SharedSavedView withModifyDate(Instant modifyDate)
+   {
+      this.modifyDate = modifyDate;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for savedViewId
+    *******************************************************************************/
+   public Integer getSavedViewId()
+   {
+      return (this.savedViewId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for savedViewId
+    *******************************************************************************/
+   public void setSavedViewId(Integer savedViewId)
+   {
+      this.savedViewId = savedViewId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for savedViewId
+    *******************************************************************************/
+   public com.kingsrook.qqq.backend.core.model.savedviews.SharedSavedView withSavedViewId(Integer savedViewId)
+   {
+      this.savedViewId = savedViewId;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for userId
+    *******************************************************************************/
+   public String getUserId()
+   {
+      return (this.userId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for userId
+    *******************************************************************************/
+   public void setUserId(String userId)
+   {
+      this.userId = userId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for userId
+    *******************************************************************************/
+   public com.kingsrook.qqq.backend.core.model.savedviews.SharedSavedView withUserId(String userId)
+   {
+      this.userId = userId;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for scope
+    *******************************************************************************/
+   public String getScope()
+   {
+      return (this.scope);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for scope
+    *******************************************************************************/
+   public void setScope(String scope)
+   {
+      this.scope = scope;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for scope
+    *******************************************************************************/
+   public com.kingsrook.qqq.backend.core.model.savedviews.SharedSavedView withScope(String scope)
+   {
+      this.scope = scope;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java

@@ -28,6 +28,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -52,9 +53,16 @@ public class QSession implements Serializable
    private Map<String, List<Serializable>> securityKeyValues;
    private Map<String, Serializable>       backendVariants;
 
-   // implementation-specific custom values
+   ///////////////////////////////////////////
+   // implementation-specific custom values //
+   ///////////////////////////////////////////
    private Map<String, String> values;
 
+   /////////////////////////////////////////////
+   // values meant to be passed to a frontend //
+   /////////////////////////////////////////////
+   private Map<String, Serializable> valuesForFrontend;
+
    public static final String VALUE_KEY_USER_TIMEZONE                = "UserTimezone";
    public static final String VALUE_KEY_USER_TIMEZONE_OFFSET_MINUTES = "UserTimezoneOffsetMinutes";
 
@@ -500,4 +508,49 @@ public class QSession implements Serializable
       return (this);
    }
 
+
+   /*******************************************************************************
+    ** Getter for valuesForFrontend
+    *******************************************************************************/
+   public Map<String, Serializable> getValuesForFrontend()
+   {
+      return (this.valuesForFrontend);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for valuesForFrontend
+    *******************************************************************************/
+   public void setValuesForFrontend(Map<String, Serializable> valuesForFrontend)
+   {
+      this.valuesForFrontend = valuesForFrontend;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for valuesForFrontend
+    *******************************************************************************/
+   public QSession withValuesForFrontend(Map<String, Serializable> valuesForFrontend)
+   {
+      this.valuesForFrontend = valuesForFrontend;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Fluent setter for a single valuesForFrontend
+    *******************************************************************************/
+   public QSession withValueForFrontend(String key, Serializable value)
+   {
+      if(this.valuesForFrontend == null)
+      {
+         this.valuesForFrontend = new LinkedHashMap<>();
+      }
+      this.valuesForFrontend.put(key, value);
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/statusmessages/DuplicateKeyBadInputStatusMessage.java

@@ -0,0 +1,39 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.statusmessages;
+
+
+/*******************************************************************************
+ ** specialization of bad-input status message, specifically for the case of
+ ** a duplicated key (e.g., unique-key validation error)
+ *******************************************************************************/
+public class DuplicateKeyBadInputStatusMessage extends BadInputStatusMessage
+{
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public DuplicateKeyBadInputStatusMessage(String message)
+   {
+      super(message);
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleCustomizerInterface.java

@@ -52,4 +52,14 @@ public interface QAuthenticationModuleCustomizerInterface
       //////////
    }
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void finalCustomizeSession(QInstance qInstance, QSession qSession)
+   {
+      //////////
+      // noop //
+      //////////
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java

@@ -230,6 +230,14 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
                }
             }
 
+            //////////////////////////////////////////////////////////////
+            // allow customizer to do custom things here, if so desired //
+            //////////////////////////////////////////////////////////////
+            if(getCustomizer() != null)
+            {
+               getCustomizer().finalCustomizeSession(qInstance, qSession);
+            }
+
             return (qSession);
          }
          else if(CollectionUtils.containsKeyWithNonNullValue(context, BASIC_AUTH_KEY))
@@ -284,7 +292,17 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
          // try to build session to see if still valid      //
          // then call method to check more session validity //
          /////////////////////////////////////////////////////
-         return buildAndValidateSession(qInstance, accessToken);
+         QSession qSession = buildAndValidateSession(qInstance, accessToken);
+
+         //////////////////////////////////////////////////////////////
+         // allow customizer to do custom things here, if so desired //
+         //////////////////////////////////////////////////////////////
+         if(getCustomizer() != null)
+         {
+            getCustomizer().finalCustomizeSession(qInstance, qSession);
+         }
+
+         return (qSession);
       }
       catch(QAuthenticationException qae)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleDispatcher.java

@@ -33,14 +33,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
  ** This class is responsible for loading a backend module, by its name, and 
  ** returning an instance.
  **
- ** TODO - make this mapping runtime-bound, not pre-compiled in.
- **
  *******************************************************************************/
 public class QBackendModuleDispatcher
 {
    private static final QLogger LOG = QLogger.getLogger(QBackendModuleDispatcher.class);
 
-   private static Map<String, String> backendTypeToModuleClassNameMap;
+   private static Map<String, String> backendTypeToModuleClassNameMap = new HashMap<>();
 
 
 
@@ -49,51 +47,6 @@ public class QBackendModuleDispatcher
     *******************************************************************************/
    public QBackendModuleDispatcher()
    {
-      initBackendTypeToModuleClassNameMap();
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static void initBackendTypeToModuleClassNameMap()
-   {
-      if(backendTypeToModuleClassNameMap != null)
-      {
-         return;
-      }
-
-      Map<String, String> newMap = new HashMap<>();
-
-      String[] moduleClassNames = new String[]
-         {
-            // todo - let modules somehow "export" their types here?
-            //  e.g., backend-core shouldn't need to "know" about the modules.
-            "com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule",
-            "com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule",
-            "com.kingsrook.qqq.backend.core.modules.backend.implementations.enumeration.EnumerationBackendModule",
-            "com.kingsrook.qqq.backend.module.rdbms.RDBMSBackendModule",
-            "com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule",
-            "com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule",
-            "com.kingsrook.qqq.backend.module.api.APIBackendModule"
-         };
-
-      for(String moduleClassName : moduleClassNames)
-      {
-         try
-         {
-            Class<?>                moduleClass = Class.forName(moduleClassName);
-            QBackendModuleInterface module      = (QBackendModuleInterface) moduleClass.getConstructor().newInstance();
-            newMap.put(module.getBackendType(), moduleClassName);
-         }
-         catch(Exception e)
-         {
-            LOG.debug("Backend module [{}] could not be loaded: {}", moduleClassName, e.getMessage());
-         }
-      }
-
-      backendTypeToModuleClassNameMap = newMap;
    }
 
 
@@ -103,7 +56,6 @@ public class QBackendModuleDispatcher
     *******************************************************************************/
    public static void registerBackendModule(QBackendModuleInterface moduleInstance)
    {
-      initBackendTypeToModuleClassNameMap();
       String backendType = moduleInstance.getBackendType();
       if(backendTypeToModuleClassNameMap.containsKey(backendType))
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleInterface.java

@@ -40,7 +40,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails
 /*******************************************************************************
  ** Interface that a QBackendModule must implement.
  **
- ** Note, some methods all have a default version, which throws a 'not implemented'
+ ** Note, all methods have a default version, which throws a 'not implemented'
  ** exception.
  **
  *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationBackendModule.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.enumerati
 import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 
 
@@ -37,6 +38,10 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
  *******************************************************************************/
 public class EnumerationBackendModule implements QBackendModuleInterface
 {
+   static
+   {
+      QBackendModuleDispatcher.registerBackendModule(new EnumerationBackendModule());
+   }
 
    /*******************************************************************************
     ** Method where a backend module must be able to provide its type (name).

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java

@@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 
 
@@ -43,6 +44,12 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
  *******************************************************************************/
 public class MemoryBackendModule implements QBackendModuleInterface
 {
+   static
+   {
+      QBackendModuleDispatcher.registerBackendModule(new MemoryBackendModule());
+   }
+
+
    /*******************************************************************************
     ** Method where a backend module must be able to provide its type (name).
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java

@@ -177,6 +177,14 @@ public class MemoryRecordStore
 
       for(QRecord qRecord : tableData)
       {
+         if(qRecord.getTableName() == null)
+         {
+            ///////////////////////////////////////////////////////////////////////////////////////////
+            // internally, doesRecordMatch likes to know table names on records, so, set if missing. //
+            ///////////////////////////////////////////////////////////////////////////////////////////
+            qRecord.setTableName(input.getTableName());
+         }
+
          boolean recordMatches = BackendQueryFilterUtils.doesRecordMatch(input.getFilter(), qRecord);
 
          if(recordMatches)
@@ -232,16 +240,7 @@ public class MemoryRecordStore
       {
          QTableMetaData      nextTable        = qInstance.getTable(queryJoin.getJoinTable());
          Collection<QRecord> nextTableRecords = getTableData(nextTable).values();
-
-         QJoinMetaData joinMetaData = Objects.requireNonNullElseGet(queryJoin.getJoinMetaData(), () ->
-         {
-            QJoinMetaData found = joinsContext.findJoinMetaData(qInstance, input.getTableName(), queryJoin.getJoinTable());
-            if(found == null)
-            {
-               throw (new RuntimeException("Could not find a join between tables [" + input.getTableName() + "][" + queryJoin.getJoinTable() + "]"));
-            }
-            return (found);
-         });
+         QJoinMetaData       joinMetaData     = Objects.requireNonNull(queryJoin.getJoinMetaData(), () -> "Could not find a join between tables [" + leftTable + "][" + queryJoin.getJoinTable() + "]");
 
          List<QRecord> nextLevelProduct = new ArrayList<>();
          for(QRecord productRecord : crossProduct)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockBackendModule.java

@@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 
 
@@ -40,6 +41,11 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
  *******************************************************************************/
 public class MockBackendModule implements QBackendModuleInterface
 {
+   static
+   {
+      QBackendModuleDispatcher.registerBackendModule(new MockBackendModule());
+   }
+
    /*******************************************************************************
     ** Method where a backend module must be able to provide its type (name).
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java

@@ -78,14 +78,16 @@ public class BackendQueryFilterUtils
          {
             ///////////////////////////////////////////////////////////////////////////////////////////////////
             // if the value isn't in the record - check, if it looks like a table.fieldName, but none of the //
-            // field names in the record are fully qualified, then just use the field-name portion...        //
+            // field names in the record are fully qualified - OR - the table name portion of the field name //
+            // matches the record's field name, then just use the field-name portion...                      //
             ///////////////////////////////////////////////////////////////////////////////////////////////////
             if(fieldName.contains("."))
             {
+               String[]                  parts  = fieldName.split("\\.");
                Map<String, Serializable> values = qRecord.getValues();
-               if(values.keySet().stream().noneMatch(n -> n.contains(".")))
+               if(values.keySet().stream().noneMatch(n -> n.contains(".")) || parts[0].equals(qRecord.getTableName()))
                {
-                  value = qRecord.getValue(fieldName.substring(fieldName.indexOf(".") + 1));
+                  value = qRecord.getValue(parts[1]);
                }
             }
          }
@@ -177,6 +179,8 @@ public class BackendQueryFilterUtils
             boolean between = (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value));
             yield !between;
          }
+         case TRUE -> true;
+         case FALSE -> false;
       };
       return criterionMatches;
    }
@@ -203,12 +207,13 @@ public class BackendQueryFilterUtils
     ** operator, update the accumulator, and if we can then short-circuit remaining
     ** operations, return a true or false.  Returning null means to keep going.
     *******************************************************************************/
-   private static Boolean applyBooleanOperator(AtomicBoolean accumulator, boolean newValue, QQueryFilter.BooleanOperator booleanOperator)
+   static Boolean applyBooleanOperator(AtomicBoolean accumulator, boolean newValue, QQueryFilter.BooleanOperator booleanOperator)
    {
       boolean accumulatorValue = accumulator.getPlain();
       if(booleanOperator.equals(QQueryFilter.BooleanOperator.AND))
       {
          accumulatorValue &= newValue;
+         accumulator.set(accumulatorValue);
          if(!accumulatorValue)
          {
             return (false);
@@ -217,6 +222,7 @@ public class BackendQueryFilterUtils
       else
       {
          accumulatorValue |= newValue;
+         accumulator.set(accumulatorValue);
          if(accumulatorValue)
          {
             return (true);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java

@@ -34,6 +34,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.DateTimeGroupBy;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
 import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
@@ -252,7 +253,7 @@ public class ColumnStatsStep implements BackendStep
 
          QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator();
          qPossibleValueTranslator.translatePossibleValuesInRecords(table, valueCounts, queryJoin == null ? null : List.of(queryJoin), null);
-         QValueFormatter.setDisplayValuesInRecords(Map.of(fieldName, field, "count", countField), valueCounts);
+         QValueFormatter.setDisplayValuesInRecords(table, Map.of(fieldName, field, "count", countField), valueCounts);
 
          runBackendStepOutput.addValue("valueCounts", valueCounts);
 
@@ -442,13 +443,13 @@ public class ColumnStatsStep implements BackendStep
             }
 
             QFieldMetaData percentField = new QFieldMetaData("percent", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.PERCENT_POINT2).withLabel("Percent");
-            QValueFormatter.setDisplayValuesInRecords(Map.of(fieldName, field, "percent", percentField), valueCounts);
+            QValueFormatter.setDisplayValuesInRecords(table, Map.of(fieldName, field, "percent", percentField), valueCounts);
          }
 
          QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(null);
          fields.forEach(qInstanceEnricher::enrichField);
 
-         QValueFormatter.setDisplayValuesInRecord(fields, statsRecord);
+         QValueFormatter.setDisplayValuesInRecord(table, fields.stream().collect(Collectors.toMap(f -> f.getName(), f -> f)), statsRecord);
 
          runBackendStepOutput.addValue("statsFields", fields);
          runBackendStepOutput.addValue("statsRecord", statsRecord);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractStep.java

@@ -112,4 +112,17 @@ public abstract class AbstractExtractStep implements BackendStep
       this.limit = limit;
    }
 
+
+
+   /*******************************************************************************
+    ** Create the record pipe to be used for this process step.
+    **
+    ** Here in case a subclass needs a different type of pipe - for example, a
+    ** DistinctFilteringRecordPipe.
+    *******************************************************************************/
+   public RecordPipe createRecordPipe(RunBackendStepInput runBackendStepInput, Integer overrideCapacity)
+   {
+      return (new RecordPipe(overrideCapacity));
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java

@@ -24,8 +24,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit
 
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.reporting.DistinctFilteringRecordPipe;
+import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -36,9 +42,13 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
@@ -105,6 +115,7 @@ public class ExtractViaQueryStep extends AbstractExtractStep
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE));
       queryInput.setFilter(filterClone);
+      getQueryJoinsForOrderByIfNeeded(queryFilter).forEach(queryJoin -> queryInput.withQueryJoin(queryJoin));
       queryInput.setSelectDistinct(true);
       queryInput.setRecordPipe(getRecordPipe());
       queryInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
@@ -139,6 +150,45 @@ public class ExtractViaQueryStep extends AbstractExtractStep
 
 
 
+   /*******************************************************************************
+    ** If the queryFilter has order-by fields from a joinTable, then create QueryJoins
+    ** for each such table - marked as LEFT, and select=true.
+    **
+    ** This is under the rationale that, the filter would have come from the frontend,
+    ** which would be doing outer-join semantics for a column being shown (but not filtered by).
+    ** If the table IS filtered by, it's still OK to do a LEFT, as we'll only get rows
+    ** that match.
+    **
+    ** Also, they are being select=true'ed so that the DISTINCT clause works (since
+    ** process queries always try to be DISTINCT).
+    *******************************************************************************/
+   private List<QueryJoin> getQueryJoinsForOrderByIfNeeded(QQueryFilter queryFilter)
+   {
+      if(queryFilter == null)
+      {
+         return (Collections.emptyList());
+      }
+
+      List<QueryJoin> rs          = new ArrayList<>();
+      Set<String>     addedTables = new HashSet<>();
+      for(QFilterOrderBy filterOrderBy : CollectionUtils.nonNullList(queryFilter.getOrderBys()))
+      {
+         if(filterOrderBy.getFieldName().contains("."))
+         {
+            String tableName = filterOrderBy.getFieldName().split("\\.")[0];
+            if(!addedTables.contains(tableName))
+            {
+               rs.add(new QueryJoin(tableName).withType(QueryJoin.Type.LEFT).withSelect(true));
+            }
+            addedTables.add(tableName);
+         }
+      }
+
+      return (rs);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -148,6 +198,7 @@ public class ExtractViaQueryStep extends AbstractExtractStep
       CountInput countInput = new CountInput();
       countInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE));
       countInput.setFilter(queryFilter);
+      getQueryJoinsForOrderByIfNeeded(queryFilter).forEach(queryJoin -> countInput.withQueryJoin(queryJoin));
       countInput.setIncludeDistinctCount(true);
       CountOutput countOutput = new CountAction().execute(countInput);
       Integer     count       = countOutput.getDistinctCount();
@@ -247,4 +298,33 @@ public class ExtractViaQueryStep extends AbstractExtractStep
       }
    }
 
+
+
+   /*******************************************************************************
+    ** Create the record pipe to be used for this process step.
+    **
+    *******************************************************************************/
+   @Override
+   public RecordPipe createRecordPipe(RunBackendStepInput runBackendStepInput, Integer overrideCapacity)
+   {
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if the filter has order-bys from a join-table, then we have to include that join-table in the SELECT clause,                 //
+      // which means we need to do distinct "manually", e.g., via a DistinctFilteringRecordPipe                                       //
+      // todo - really, wouldn't this only be if it's a many-join?  but that's not completely trivial to detect, given join-chains... //
+      //  as it is, we may end up using DistinctPipe in some cases that we need it - which isn't an error, just slightly sub-optimal. //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      List<QueryJoin> queryJoinsForOrderByIfNeeded = getQueryJoinsForOrderByIfNeeded(queryFilter);
+      boolean         needDistinctPipe             = CollectionUtils.nullSafeHasContents(queryJoinsForOrderByIfNeeded);
+
+      if(needDistinctPipe)
+      {
+         String         sourceTableName = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE);
+         QTableMetaData sourceTable     = runBackendStepInput.getInstance().getTable(sourceTableName);
+         return (new DistinctFilteringRecordPipe(new UniqueKey(sourceTable.getPrimaryKeyField()), overrideCapacity));
+      }
+      else
+      {
+         return (super.createRecordPipe(runBackendStepInput, overrideCapacity));
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java

@@ -72,15 +72,6 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
          return;
       }
 
-      //////////////////////////////
-      // set up the extract steps //
-      //////////////////////////////
-      AbstractExtractStep extractStep = getExtractStep(runBackendStepInput);
-      RecordPipe          recordPipe  = new RecordPipe();
-      extractStep.setLimit(limit);
-      extractStep.setRecordPipe(recordPipe);
-      extractStep.preRun(runBackendStepInput, runBackendStepOutput);
-
       /////////////////////////////////////////////////////////////////
       // if we're running inside an automation, then skip this step. //
       /////////////////////////////////////////////////////////////////
@@ -90,6 +81,19 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
          return;
       }
 
+      /////////////////////////////
+      // set up the extract step //
+      /////////////////////////////
+      AbstractExtractStep extractStep = getExtractStep(runBackendStepInput);
+      extractStep.setLimit(limit);
+      extractStep.preRun(runBackendStepInput, runBackendStepOutput);
+
+      //////////////////////////////////////////
+      // set up a record pipe for the process //
+      //////////////////////////////////////////
+      RecordPipe recordPipe = extractStep.createRecordPipe(runBackendStepInput, null);
+      extractStep.setRecordPipe(recordPipe);
+
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // if skipping frontend steps, skip this action -                                                                 //
       // but, if inside an (ideally, only async) API call, at least do the count, so status calls can get x of y status //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java

@@ -77,14 +77,16 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back
       //////////////////////////////////////////////////////////////////////////////////////////////////////////////
       moveReviewStepAfterValidateStep(runBackendStepOutput);
 
+      AbstractExtractStep   extractStep   = getExtractStep(runBackendStepInput);
+      AbstractTransformStep transformStep = getTransformStep(runBackendStepInput);
+
+      runBackendStepInput.getAsyncJobCallback().updateStatus("Validating Records");
+
       //////////////////////////////////////////////////////////
       // basically repeat the preview step, but with no limit //
       //////////////////////////////////////////////////////////
       runBackendStepInput.getAsyncJobCallback().updateStatus("Validating Records");
 
-      AbstractExtractStep extractStep = getExtractStep(runBackendStepInput);
-      AbstractTransformStep transformStep = getTransformStep(runBackendStepInput);
-
       //////////////////////////////////////////////////////////////////////
       // let the transform step override the capacity for the record pipe //
       //////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mock/MockBackendStep.java

@@ -56,7 +56,7 @@ public class MockBackendStep implements BackendStep
 
       runBackendStepInput.getRecords().forEach(r ->
       {
-         LOG.info("We are mocking {}: {}", r.getValueString("firstName"), r.getValue(FIELD_MOCK_VALUE));
+         LOG.info("We are mocking " + r.getValueString("firstName") + ": " + r.getValue(FIELD_MOCK_VALUE));
          r.setValue(FIELD_MOCK_VALUE, "Ha ha!");
          r.setValue("greetingMessage", runBackendStepInput.getValueString(FIELD_GREETING_PREFIX) + " " + r.getValueString("firstName") + " " + runBackendStepInput.getValueString(FIELD_GREETING_SUFFIX));
       });

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java

@@ -55,10 +55,11 @@ import com.kingsrook.qqq.backend.core.model.savedreports.RenderedReportStatus;
 import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport;
 import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
- **
+ ** Process step to actually execute rendering a saved report.
  *******************************************************************************/
 public class RenderSavedReportExecuteStep implements BackendStep
 {
@@ -86,6 +87,7 @@ public class RenderSavedReportExecuteStep implements BackendStep
          String       storageReference     = LocalDate.now() + "/" + LocalTime.now().toString().replaceAll(":", "").replaceFirst("\\..*", "") + "/" + UUID.randomUUID() + "/" + downloadFileBaseName + "." + reportFormat.getExtension();
          OutputStream outputStream         = new StorageAction().createOutputStream(new StorageInput(storageTableName).withReference(storageReference));
 
+         LOG.info("Starting to render a report", logPair("savedReportId", savedReport.getId()), logPair("tableName", savedReport.getTableName()), logPair("storageReference", storageReference));
          runBackendStepInput.getAsyncJobCallback().updateStatus("Generating Report");
 
          //////////////////////////////////////////////////////////////////
@@ -133,6 +135,7 @@ public class RenderSavedReportExecuteStep implements BackendStep
          runBackendStepOutput.addValue("downloadFileName", downloadFileBaseName + "." + reportFormat.getExtension());
          runBackendStepOutput.addValue("storageTableName", storageTableName);
          runBackendStepOutput.addValue("storageReference", storageReference);
+         LOG.info("Completed rendering a report", logPair("savedReportId", savedReport.getId()), logPair("tableName", savedReport.getTableName()), logPair("storageReference", storageReference), logPair("rowCount", reportOutput.getTotalRecordCount()));
       }
       catch(Exception e)
       {
@@ -167,7 +170,10 @@ public class RenderSavedReportExecuteStep implements BackendStep
          downloadFileBaseName = report.getLabel();
       }
 
-      downloadFileBaseName = downloadFileBaseName.replaceAll("/", "-");
+      //////////////////////////////////////////////////
+      // these chars have caused issues, so, disallow //
+      //////////////////////////////////////////////////
+      downloadFileBaseName = downloadFileBaseName.replaceAll("/", "-").replaceAll(",", "_");
 
       return (downloadFileBaseName + " - " + datePart);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java

@@ -41,7 +41,7 @@ import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport;
 
 
 /*******************************************************************************
- **
+ ** define process for rendering saved reports!
  *******************************************************************************/
 public class RenderSavedReportMetaDataProducer implements MetaDataProducerInterface<QProcessMetaData>
 {
@@ -60,6 +60,7 @@ public class RenderSavedReportMetaDataProducer implements MetaDataProducerInterf
    {
       QProcessMetaData process = new QProcessMetaData()
          .withName(NAME)
+         .withLabel("Render Report")
          .withTableName(SavedReport.TABLE_NAME)
          .withIcon(new QIcon().withName("print"))
          .addStep(new QBackendStepMetaData()
@@ -79,7 +80,6 @@ public class RenderSavedReportMetaDataProducer implements MetaDataProducerInterf
             .withInputData(new QFunctionInputMetaData().withRecordListMetaData(new QRecordListMetaData()
                .withTableName(SavedReport.TABLE_NAME)))
             .withCode(new QCodeReference(RenderSavedReportExecuteStep.class)))
-         // todo - no no, stream the damn thing... how to do that??
          .addStep(new QFrontendStepMetaData()
             .withName("output")
             .withComponent(new QFrontendComponentMetaData().withType(QComponentType.DOWNLOAD_FORM)));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportPreStep.java

@@ -36,7 +36,9 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
- **
+ ** initial backend-step before rendering a saved report.  does some basic
+ ** validations, and then (in future) will set up input fields (how??) for the
+ ** input screen.
  *******************************************************************************/
 public class RenderSavedReportPreStep implements BackendStep
 {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java

@@ -22,11 +22,14 @@
 package com.kingsrook.qqq.backend.core.processes.implementations.savedreports;
 
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.function.Consumer;
 import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
@@ -54,17 +57,21 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
-import org.apache.commons.lang.BooleanUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
+ ** GenerateReportAction takes in ReportMetaData.
  **
+ ** This class knows how to adapt from a SavedReport to a ReportMetaData, so that
+ ** we can render a saved report.
  *******************************************************************************/
 public class SavedReportToReportMetaDataAdapter
 {
    private static final QLogger LOG = QLogger.getLogger(SavedReportToReportMetaDataAdapter.class);
 
+   private static Consumer<ObjectMapper> jsonMapperCustomizer = om -> om.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+
 
 
    /*******************************************************************************
@@ -89,7 +96,7 @@ public class SavedReportToReportMetaDataAdapter
 
          QTableMetaData table = qInstance.getTable(savedReport.getTableName());
          dataSource.setSourceTable(savedReport.getTableName());
-         dataSource.setQueryFilter(JsonUtils.toObject(savedReport.getQueryFilterJson(), QQueryFilter.class));
+         dataSource.setQueryFilter(getQQueryFilter(savedReport.getQueryFilterJson()));
 
          //////////////////////////
          // set up the main view //
@@ -99,7 +106,7 @@ public class SavedReportToReportMetaDataAdapter
          view.setName("main");
          view.setType(ReportType.TABLE);
          view.setDataSourceName(dataSource.getName());
-         view.setLabel(savedReport.getLabel()); // todo eh?
+         view.setLabel(savedReport.getLabel());
          view.setIncludeHeaderRow(true);
 
          ///////////////////////////////////////////////////////////////////////////////////////////////
@@ -107,23 +114,15 @@ public class SavedReportToReportMetaDataAdapter
          // map them to a list of QReportField objects                                                //
          // also keep track of what joinTables we find that we need to select                         //
          ///////////////////////////////////////////////////////////////////////////////////////////////
-         ReportColumns columnsObject = JsonUtils.toObject(savedReport.getColumnsJson(), ReportColumns.class, om -> om.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES));
+         ReportColumns columnsObject = getReportColumns(savedReport.getColumnsJson());
 
          List<QReportField> reportColumns = new ArrayList<>();
          view.setColumns(reportColumns);
 
          Set<String> neededJoinTables = new HashSet<>();
 
-         for(ReportColumn column : columnsObject.getColumns())
+         for(ReportColumn column : columnsObject.extractVisibleColumns())
          {
-            ////////////////////////////////////////////////////////////////////////////////////////////////////
-            // if isVisible is missing, we assume it to be true - so only if it isFalse do we skip the column //
-            ////////////////////////////////////////////////////////////////////////////////////////////////////
-            if(BooleanUtils.isFalse(column.getIsVisible()))
-            {
-               continue;
-            }
-
             ////////////////////////////////////////////////////
             // figure out the field being named by the column //
             ////////////////////////////////////////////////////
@@ -179,7 +178,7 @@ public class SavedReportToReportMetaDataAdapter
          /////////////////////////////////////////
          if(StringUtils.hasContent(savedReport.getPivotTableJson()))
          {
-            PivotTableDefinition pivotTableDefinition = JsonUtils.toObject(savedReport.getPivotTableJson(), PivotTableDefinition.class);
+            PivotTableDefinition pivotTableDefinition = getPivotTableDefinition(savedReport.getPivotTableJson());
 
             QReportView pivotView = new QReportView();
             reportMetaData.getViews().add(pivotView);
@@ -271,7 +270,7 @@ public class SavedReportToReportMetaDataAdapter
             ////////////////////////////////////
             // todo turn on when implementing //
             ////////////////////////////////////
-            // reportMetaData.setInputFields(JsonUtils.toObject(savedReport.getInputFieldsJson(), new TypeReference<>() {}));
+            // reportMetaData.setInputFields(JsonUtils.toObject(savedReport.getInputFieldsJson(), new TypeReference<>() {}), objectMapperConsumer);
             throw (new IllegalStateException("Input Fields are not yet implemented"));
          }
 
@@ -286,6 +285,36 @@ public class SavedReportToReportMetaDataAdapter
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static PivotTableDefinition getPivotTableDefinition(String pivotTableJson) throws IOException
+   {
+      return JsonUtils.toObject(pivotTableJson, PivotTableDefinition.class, jsonMapperCustomizer);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static ReportColumns getReportColumns(String columnsJson) throws IOException
+   {
+      return JsonUtils.toObject(columnsJson, ReportColumns.class, jsonMapperCustomizer);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QQueryFilter getQQueryFilter(String queryFilterJson) throws IOException
+   {
+      return JsonUtils.toObject(queryFilterJson, QQueryFilter.class, jsonMapperCustomizer);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -322,7 +351,6 @@ public class SavedReportToReportMetaDataAdapter
 
 
 
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/DeleteSharedRecordProcess.java

@@ -0,0 +1,130 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.sharing;
+
+
+import java.util.List;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ ** DeleteSharedRecord:  {tableName; recordId; shareId;}
+ *******************************************************************************/
+public class DeleteSharedRecordProcess implements BackendStep, MetaDataProducerInterface<QProcessMetaData>
+{
+   public static final String NAME = "deleteSharedRecord";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QProcessMetaData produce(QInstance qInstance) throws QException
+   {
+      return new QProcessMetaData()
+         .withName(NAME)
+         .withIcon(new QIcon().withName("share"))
+         .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED)) // todo confirm or protect
+         .withStepList(List.of(
+            new QBackendStepMetaData()
+               .withName("execute")
+               .withCode(new QCodeReference(getClass()))
+               .withInputData(new QFunctionInputMetaData()
+                  .withField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)) // todo - actually only a subset of this...
+                  .withField(new QFieldMetaData("recordId", QFieldType.STRING))
+                  .withField(new QFieldMetaData("shareId", QFieldType.INTEGER))
+               )
+         ));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      String tableName        = runBackendStepInput.getValueString("tableName");
+      String recordIdString   = runBackendStepInput.getValueString("recordId");
+      Integer shareId         = runBackendStepInput.getValueInteger("shareId");
+
+      Objects.requireNonNull(tableName, "Missing required input: tableName");
+      Objects.requireNonNull(recordIdString, "Missing required input: recordId");
+      Objects.requireNonNull(shareId, "Missing required input: shareId");
+
+      try
+      {
+         SharedRecordProcessUtils.AssetTableAndRecord assetTableAndRecord = SharedRecordProcessUtils.getAssetTableAndRecord(tableName, recordIdString);
+
+         ShareableTableMetaData shareableTableMetaData = assetTableAndRecord.shareableTableMetaData();
+         QRecord                assetRecord            = assetTableAndRecord.record();
+
+         SharedRecordProcessUtils.assertRecordOwnership(shareableTableMetaData, assetRecord, "delete shares of");
+
+         ///////////////////
+         // do the delete //
+         ///////////////////
+         DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(shareableTableMetaData.getSharedRecordTableName()).withPrimaryKeys(List.of(shareId)));
+
+         //////////////////////
+         // check for errors //
+         //////////////////////
+         if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors()))
+         {
+            throw (new QException("Error deleting shared record: " + deleteOutput.getRecordsWithErrors().get(0).getErrors().get(0).getMessage()));
+         }
+      }
+      catch(QException qe)
+      {
+         throw (qe);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error deleting shared record", e));
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/EditSharedRecordProcess.java

@@ -0,0 +1,140 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.sharing;
+
+
+import java.util.List;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ ** EditSharedRecord:  {tableName; recordId; shareId; scopeId;}
+ *******************************************************************************/
+public class EditSharedRecordProcess implements BackendStep, MetaDataProducerInterface<QProcessMetaData>
+{
+   public static final String NAME = "editSharedRecord";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QProcessMetaData produce(QInstance qInstance) throws QException
+   {
+      return new QProcessMetaData()
+         .withName(NAME)
+         .withIcon(new QIcon().withName("share"))
+         .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED)) // todo confirm or protect
+         .withStepList(List.of(
+            new QBackendStepMetaData()
+               .withName("execute")
+               .withCode(new QCodeReference(getClass()))
+               .withInputData(new QFunctionInputMetaData()
+                  .withField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)) // todo - actually only a subset of this...
+                  .withField(new QFieldMetaData("recordId", QFieldType.STRING))
+                  .withField(new QFieldMetaData("scopeId", QFieldType.STRING).withPossibleValueSourceName(ShareScopePossibleValueMetaDataProducer.NAME))
+                  .withField(new QFieldMetaData("shareId", QFieldType.INTEGER))
+               )
+         ));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      String  tableName      = runBackendStepInput.getValueString("tableName");
+      String  recordIdString = runBackendStepInput.getValueString("recordId");
+      String  scopeId        = runBackendStepInput.getValueString("scopeId");
+      Integer shareId        = runBackendStepInput.getValueInteger("shareId");
+
+      Objects.requireNonNull(tableName, "Missing required input: tableName");
+      Objects.requireNonNull(recordIdString, "Missing required input: recordId");
+      Objects.requireNonNull(scopeId, "Missing required input: scopeId");
+      Objects.requireNonNull(shareId, "Missing required input: shareId");
+
+      try
+      {
+         SharedRecordProcessUtils.AssetTableAndRecord assetTableAndRecord = SharedRecordProcessUtils.getAssetTableAndRecord(tableName, recordIdString);
+
+         ShareableTableMetaData shareableTableMetaData = assetTableAndRecord.shareableTableMetaData();
+         QRecord                assetRecord            = assetTableAndRecord.record();
+         QTableMetaData         shareTable             = QContext.getQInstance().getTable(shareableTableMetaData.getSharedRecordTableName());
+
+         SharedRecordProcessUtils.assertRecordOwnership(shareableTableMetaData, assetRecord, "edit shares of");
+         ShareScope shareScope = SharedRecordProcessUtils.validateScopeId(scopeId);
+
+         ///////////////////
+         // do the insert //
+         ///////////////////
+         UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(shareableTableMetaData.getSharedRecordTableName()).withRecord(new QRecord()
+            .withValue(shareTable.getPrimaryKeyField(), shareId)
+            .withValue(shareableTableMetaData.getScopeFieldName(), shareScope.getPossibleValueId())));
+
+         //////////////////////
+         // check for errors //
+         //////////////////////
+         if(CollectionUtils.nullSafeHasContents(updateOutput.getRecords().get(0).getErrors()))
+         {
+            throw (new QException("Error editing shared record: " + updateOutput.getRecords().get(0).getErrors().get(0).getMessage()));
+         }
+      }
+      catch(QException qe)
+      {
+         throw (qe);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error editing shared record", e));
+      }
+   }
+
+}