CE-1107: added ability to use replaceAction specifying that null key values be treated as equal

Feature/ce 978 crashing nodes

.circleci/config.yml

@@ -98,6 +98,31 @@ commands:
             - ~/.m2
           key: v1-dependencies-{{ checksum "pom.xml" }}
 
+  install_asciidoctor:
+    steps:
+      - checkout
+      - run:
+          name: Install asciidoctor
+          command: |
+            sudo apt-get update
+            sudo apt install -y asciidoctor
+
+  run_asciidoctor:
+    steps:
+      - run:
+          name: Run asciidoctor
+          command: |
+            cd docs
+            asciidoctor -a docinfo=shared index.adoc
+
+  upload_docs_site:
+    steps:
+      - run:
+          name: scp html to justinsgotskinnylegs.com
+          command: |
+            cd docs
+            scp index.html dkelkhoff@45.79.44.221:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html
+
 jobs:
   mvn_test:
     executor: localstack/default
@@ -114,6 +139,13 @@ jobs:
       - mvn_verify
       - mvn_jar_deploy
 
+  publish_asciidoc:
+    executor: localstack/default
+    steps:
+      - install_asciidoctor
+      - run_asciidoctor
+      - upload_docs_site
+
 workflows:
   test_only:
     jobs:
@@ -134,4 +166,7 @@ workflows:
               only: /dev/
             tags:
               only: /(version|snapshot)-.*/
-
+      - publish_asciidoc:
+          filters:
+            branches:
+              only: /dev/

docs/metaData/Fields.adoc

@@ -21,5 +21,99 @@ Used to set values in the `displayValues` map within a `QRecord`.
 * `possibleValueSourceName` - *String* - Reference to a {link-pvs} to be used for this field.
 Values in this field should correspond to ids from the referenced Possible Value Source.
 * `maxLength` - *Integer* - Maximum length (number of characters) allowed for values in this field.
-Only applicable for fields with `type=STRING`.
+Only applicable for fields with `type=STRING`.  Needs to be used with a `FieldBehavior` of type `ValueTooLongBehavior`.
-* `
+
+==== Field Behaviors
+Additional behaviors can be attached to fields through the use of the `behaviors` attribute,
+which is a `Set` of 0 or more instances of implementations of the `FieldBehavior` interface.
+Note that in some cases, these instances may be `enum` constants,
+but other times may be regular Objects.
+
+QQQ provides a set of common field behaviors.
+Applications can also define their own field behaviors by implementing the `FieldBehavior` interface,
+and attaching instances of their custom behavior classes to fields.
+
+===== ValueTooLongBehavior
+Used on String fields.  Requires the field to have a `maxLength` set.
+Depending on the chosen instance of this enum, before a record is Inserted or Updated,
+if the value in the field is longer than the `maxLength`, then one of the following actions can occur:
+
+* `TRUNCATE` - The value will be simply truncated to the `maxLength`.
+* `TRUNCATE_ELLIPSIS` - The value will be truncated to 3 characters less than the `maxLength`, and three periods (an ellipsis) will be placed at the end.
+* `ERROR` - An error will be reported, and the record will not be inserted or updated.
+* `PASS_THROUGH` - Nothing will happen.  This is the same as not having a `ValueTooLongBehavior` on the field.
+
+[source,java]
+.Examples of using ValueTooLongBehavior
+----
+   new QFieldMetaData("sku", QFieldType.STRING)
+      .withMaxLength(40),
+      .withBehavior(ValueTooLongBehavior.ERROR),
+
+   new QFieldMetaData("reason", QFieldType.STRING)
+      .withMaxLength(250),
+      .withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS),
+
+----
+
+===== DynamicDefaultValueBehavior
+Used to set a dynamic default value to a field when it is being inserted or updated.
+For example, instead of having a hard-coded `defaultValue` specified in the field meta-data,
+and instead of having to add, for example, a pre-insert custom action.
+
+* `CREATE_DATE` - On inserts, sets the field's value to the current time.
+* `MODIFY_DATE` - On inserts and updates, sets the field's value to the current time.
+* `USER_ID` - On inserts and updates, sets the field's value to the current user's id (but only if the value is currently null).
+
+_Note that the `QInstanceEnricher` will, by default, add the `CREATE_DATE` and `MODIFY_DATE` `DynamicDefaultValueBehavior`
+options to any fields named `"createDate"` or `"modifyDate"`.
+This behavior can be disabled by setting the `configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate` property
+on the `QInstanceEnricher` instance used by the application to `false`._
+
+[source,java]
+.Examples of using DynamicDefaultValueBehavior
+----
+   new QFieldMetaData("createDate", QFieldType.DATE_TIME)
+      .withBehavior(DynamicDefaultValueBehavior.CREATE_DATE),
+
+   new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)
+      .withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE),
+
+   new QFieldMetaData("createdByUserId", QFieldType.STRING)
+      .withBehavior(DynamicDefaultValueBehavior.USER_ID),
+----
+
+===== DateTimeDisplayValueBehavior
+By default, in QQQ, fields of type `DATE_TIME` are stored in UTC,
+and their values in a QRecord is a java `Instant` instance, which is always UTC.
+However, frontends will prefer to display date-time values in the user's local Time Zone whenever possible.
+
+Using `DateTimeDisplayValueBehavior` allows a `DATE_TIME` field to be displayed in a different Time Zone.
+An example use-case for this would be displaying airplane flight times,
+where you would want a flight from California to New York to display Pacific Time for its departure time,
+and Eastern Time for its arrival.
+
+An instance of `DateTimeDisplayValueBehavior` can be configured to either use a hard-coded time `ZoneId`
+(for example, to always show users UTC, or a business's home-office time zone).
+Or, it can be set up to get the time zone to use from another field in the table.
+
+[source,java]
+.Examples of using DateTimeDisplayValueBehavior
+----
+new QTableMetaData().withName("flights").withFields(List.of(
+   ...
+   new QFieldMetaData("departureTimeZoneId", QFieldType.STRING),
+   new QFieldMetaData("arrivaTimeZoneId", QFieldType.STRING),
+
+   new QFieldMetaData("departureTime", QFieldType.DATE_TIME)
+      .withBehavior(new DateTimeDisplayValueBehavior()
+         .withZoneIdFromFieldName("departureTimeZoneId")),
+
+   new QFieldMetaData("arrivalTime", QFieldType.DATE_TIME)
+      .withBehavior(new DateTimeDisplayValueBehavior()
+         .withZoneIdFromFieldName("arrivalTimeZoneId"))
+
+   new QFieldMetaData("ticketSaleStartDateTime", QFieldType.DATE_TIME)
+      .withBehavior(new DateTimeDisplayValueBehavior()
+         .withDefaultZoneId("UTC"))
+----

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -526,7 +526,7 @@ public class PollingAutomationPerTableRunner implements Runnable
          // note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
          //////////////////////////////////////////////////////////////////////////////////////////////////////////////
          List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
-         LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
+         LOG.debug("Of the [" + records.size() + "] records that were pending automations, [" + matchingQRecords.size() + "] of them match the filter on the action:" +  action);
          if(CollectionUtils.nullSafeHasContents(matchingQRecords))
          {
             LOG.debug("  Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
@@ -601,7 +601,7 @@ public class PollingAutomationPerTableRunner implements Runnable
 
    /*******************************************************************************
     ** Finally, actually run action code against a list of known matching records.
-    ** todo not commit - move to somewhere genericer
+    **
     *******************************************************************************/
    public static void applyActionToMatchingRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action) throws Exception
    {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java

@@ -96,7 +96,7 @@ public class QCodeLoader
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //
@@ -135,7 +135,7 @@ public class QCodeLoader
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //
@@ -187,7 +187,7 @@ public class QCodeLoader
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/AbstractWidgetRenderer.java

@@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -72,80 +73,104 @@ public abstract class AbstractWidgetRenderer
     *******************************************************************************/
    protected boolean setupDropdowns(RenderWidgetInput input, QWidgetMetaData metaData, QWidgetData widgetData) throws QException
    {
-      List<List<Map<String, String>>> pvsData                   = new ArrayList<>();
+      List<List<Map<String, String>>> dataList                  = new ArrayList<>();
-      List<String>                    pvsLabels                 = new ArrayList<>();
+      List<String>                    labelList                 = new ArrayList<>();
-      List<String>                    pvsNames                  = new ArrayList<>();
+      List<String>                    nameList                  = new ArrayList<>();
       List<String>                    missingRequiredSelections = new ArrayList<>();
       for(WidgetDropdownData dropdownData : CollectionUtils.nonNullList(metaData.getDropdowns()))
       {
-         String               possibleValueSourceName = dropdownData.getPossibleValueSourceName();
+         if(WidgetDropdownType.DATE_PICKER.equals(dropdownData.getType()))
-         QPossibleValueSource possibleValueSource     = input.getInstance().getPossibleValueSource(possibleValueSourceName);
-
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // this looks complicated, but is just look for a label in the dropdown data and if found use it,                                                                                  //
-         // otherwise look for label in PVS and if found use that, otherwise just use the PVS name                                                                                          //
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName);
-         pvsLabels.add(pvsLabel);
-         pvsNames.add(possibleValueSourceName);
-
-         SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput();
-         pvsInput.setPossibleValueSourceName(possibleValueSourceName);
-
-         if(dropdownData.getForeignKeyFieldName() != null)
          {
-            ////////////////////////////////////////
+            String name = dropdownData.getName();
-            // look for an id in the query params //
+            nameList.add(name);
-            ////////////////////////////////////////
+            labelList.add(dropdownData.getLabel());
-            Integer id = null;
+            dataList.add(new ArrayList<>());
-            if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id")))
+
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // sure that something has been selected, and if not, display a message that a selection needs made       //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(dropdownData.getIsRequired())
             {
-               id = Integer.parseInt(input.getQueryParams().get("id"));
+               if(!input.getQueryParams().containsKey(name) || !StringUtils.hasContent(input.getQueryParams().get(name)))
-            }
+               {
-            if(id != null)
+                  missingRequiredSelections.add(dropdownData.getLabel());
-            {
+               }
-               pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria(
-                  new QFilterCriteria(
-                     dropdownData.getForeignKeyFieldName(),
-                     QCriteriaOperator.EQUALS,
-                     id)));
             }
          }
-
+         else
-         SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput);
-
-         List<Map<String, String>> dropdownOptionList = new ArrayList<>();
-         pvsData.add(dropdownOptionList);
-
-         //////////////////////////////////////////
-         // sort results, dedupe, and add to map //
-         //////////////////////////////////////////
-         Set<String> exists = new HashSet<>();
-         output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel()));
-         for(QPossibleValue<?> possibleValue : output.getResults())
          {
-            dropdownOptionList.add(MapBuilder.of(
+            String possibleValueSourceName = dropdownData.getPossibleValueSourceName();
-               "id", String.valueOf(possibleValue.getId()),
+            if(possibleValueSourceName != null)
-               "label", possibleValue.getLabel()
-            ));
-         }
-
-         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // because we know the dropdowns and what the field names will be when something is selected, we can make //
-         // sure that something has been selected, and if not, display a message that a selection needs made       //
-         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         if(dropdownData.getIsRequired())
-         {
-            if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName)))
             {
-               missingRequiredSelections.add(pvsLabel);
+               QPossibleValueSource possibleValueSource = input.getInstance().getPossibleValueSource(possibleValueSourceName);
+
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // this looks complicated, but is just look for a label in the dropdown data and if found use it,                                                                                  //
+               // otherwise look for label in PVS and if found use that, otherwise just use the PVS name                                                                                          //
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName);
+               labelList.add(pvsLabel);
+               nameList.add(possibleValueSourceName);
+
+               SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput();
+               pvsInput.setPossibleValueSourceName(possibleValueSourceName);
+
+               if(dropdownData.getForeignKeyFieldName() != null)
+               {
+                  ////////////////////////////////////////
+                  // look for an id in the query params //
+                  ////////////////////////////////////////
+                  Integer id = null;
+                  if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id")))
+                  {
+                     id = Integer.parseInt(input.getQueryParams().get("id"));
+                  }
+                  if(id != null)
+                  {
+                     pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria(
+                        new QFilterCriteria(
+                           dropdownData.getForeignKeyFieldName(),
+                           QCriteriaOperator.EQUALS,
+                           id)));
+                  }
+               }
+
+               SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput);
+
+               List<Map<String, String>> dropdownOptionList = new ArrayList<>();
+               dataList.add(dropdownOptionList);
+
+               //////////////////////////////////////////
+               // sort results, dedupe, and add to map //
+               //////////////////////////////////////////
+               Set<String> exists = new HashSet<>();
+               output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel()));
+               for(QPossibleValue<?> possibleValue : output.getResults())
+               {
+                  dropdownOptionList.add(MapBuilder.of(
+                     "id", String.valueOf(possibleValue.getId()),
+                     "label", possibleValue.getLabel()
+                  ));
+               }
+
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // because we know the dropdowns and what the field names will be when something is selected, we can make //
+               // sure that something has been selected, and if not, display a message that a selection needs made       //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(dropdownData.getIsRequired())
+               {
+                  if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName)))
+                  {
+                     missingRequiredSelections.add(pvsLabel);
+                  }
+               }
             }
          }
       }
 
-      widgetData.setDropdownNameList(pvsNames);
+      widgetData.setDropdownNameList(nameList);
-      widgetData.setDropdownLabelList(pvsLabels);
+      widgetData.setDropdownLabelList(labelList);
-      widgetData.setDropdownDataList(pvsData);
+      widgetData.setDropdownDataList(dataList);
 
       ////////////////////////////////////////////////////////////////////////////////
       // if there are any missing required dropdowns, build up a message to display //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java

@@ -232,6 +232,7 @@ public class ExportAction
       }
       queryInput.getFilter().setLimit(exportInput.getLimit());
       queryInput.setShouldTranslatePossibleValues(true);
+      queryInput.withQueryHint(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
 
       /////////////////////////////////////////////////////////////////
       // tell this query that it needs to put its output into a pipe //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java

@@ -138,7 +138,7 @@ public class RecordPipe
          {
             if(now - sleepLoopStartTime > MAX_SLEEP_LOOP_MILLIS)
             {
-               LOG.warn("Giving up adding record to pipe, due to pipe being full for more than {} millis", MAX_SLEEP_LOOP_MILLIS);
+               LOG.warn("Giving up adding record to pipe, due to pipe being full for more than " + MAX_SLEEP_LOOP_MILLIS + " millis");
                throw (new IllegalStateException("Giving up adding record to pipe, due to pipe staying full too long."));
             }
             LOG.trace("Record pipe.add failed (due to full pipe).  Blocking.");

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java

@@ -47,6 +47,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import org.apache.commons.lang.BooleanUtils;
 
 
 /*******************************************************************************
@@ -79,9 +80,11 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
 
       try
       {
-         QTableMetaData table           = input.getTable();
+         QTableMetaData table                     = input.getTable();
-         UniqueKey      uniqueKey       = input.getKey();
+         UniqueKey      uniqueKey                 = input.getKey();
-         String         primaryKeyField = table.getPrimaryKeyField();
+         String         primaryKeyField           = table.getPrimaryKeyField();
+         boolean        allowNullKeyValuesToEqual = BooleanUtils.isTrue(input.getAllowNullKeyValuesToEqual());
+
          if(transaction == null)
          {
             transaction = QBackendTransaction.openFor(new InsertInput(input.getTableName()));
@@ -98,10 +101,11 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
             // originally it was thought that we'd need to pass the filter in here           //
             // but, it's been decided not to.  the filter only applies to what we can delete //
             ///////////////////////////////////////////////////////////////////////////////////
-            Map<List<Serializable>, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey);
+            Map<List<Serializable>, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey, allowNullKeyValuesToEqual);
+
             for(QRecord record : page)
             {
-               Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
+               Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
                if(keyValues.isPresent())
                {
                   if(existingKeys.containsKey(keyValues.get()))

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java

@@ -54,7 +54,7 @@ public class UniqueKeyHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static Map<List<Serializable>, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List<QRecord> recordList, UniqueKey uniqueKey) throws QException
+   public static Map<List<Serializable>, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List<QRecord> recordList, UniqueKey uniqueKey, boolean allowNullKeyValuesToEqual) throws QException
    {
       List<String>                          ukFieldNames    = uniqueKey.getFieldNames();
       Map<List<Serializable>, Serializable> existingRecords = new HashMap<>();
@@ -112,7 +112,7 @@ public class UniqueKeyHelper
          QueryOutput queryOutput = new QueryAction().execute(queryInput);
          for(QRecord record : queryOutput.getRecords())
          {
-            Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record);
+            Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
             if(keyValues.isPresent())
             {
                existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField()));
@@ -128,7 +128,17 @@ public class UniqueKeyHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static Optional<List<Serializable>> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record)
+   public static Map<List<Serializable>, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List<QRecord> recordList, UniqueKey uniqueKey) throws QException
+   {
+      return (getExistingKeys(transaction, table, recordList, uniqueKey, false));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Optional<List<Serializable>> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record, boolean allowNullKeyValuesToEqual)
    {
       try
       {
@@ -138,7 +148,19 @@ public class UniqueKeyHelper
             QFieldMetaData field      = table.getField(fieldName);
             Serializable   value      = record.getValue(fieldName);
             Serializable   typedValue = ValueUtils.getValueAsFieldType(field.getType(), value);
-            keyValues.add(typedValue == null ? new NullUniqueKeyValue() : typedValue);
+
+            ///////////////////////////////////////////////////////////////////////////////////
+            // if null value, look at flag to determine if a null should be used (which will //
+            // allow keys to match), or a NullUniqueKeyValue, (which will never match)       //
+            ///////////////////////////////////////////////////////////////////////////////////
+            if(typedValue == null)
+            {
+               keyValues.add(allowNullKeyValuesToEqual ? null : new NullUniqueKeyValue());
+            }
+            else
+            {
+               keyValues.add(typedValue);
+            }
          }
          return (Optional.of(keyValues));
       }
@@ -150,6 +172,16 @@ public class UniqueKeyHelper
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Optional<List<Serializable>> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record)
+   {
+      return (getKeyValues(table, uniqueKey, record, false));
+   }
+
+
+
    /*******************************************************************************
     ** To make a list of unique key values here behave like they do in an RDBMS
     ** (which is what we're trying to mimic - which is - 2 null values in a field

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java

@@ -28,7 +28,6 @@ import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -364,7 +363,9 @@ public class QValueFormatter
             }
          }
 
-         setDisplayValuesInRecord(fieldMap, record);
+         ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, QContext.getQInstance(), table, records, null);
+
+         setDisplayValuesInRecord(table, fieldMap, record, true);
          record.setRecordLabel(formatRecordLabel(table, record));
       }
    }
@@ -374,61 +375,49 @@ public class QValueFormatter
    /*******************************************************************************
     ** For a list of records, set their recordLabels and display values
     *******************************************************************************/
-   public static void setDisplayValuesInRecords(Collection<QFieldMetaData> fields, List<QRecord> records)
+   public static void setDisplayValuesInRecords(QTableMetaData table, Map<String, QFieldMetaData> fields, List<QRecord> records)
    {
       if(records == null)
       {
          return;
       }
 
+      if(table != null)
+      {
+         ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, QContext.getQInstance(), table, records, null);
+      }
+
       for(QRecord record : records)
       {
-         setDisplayValuesInRecord(fields, record);
+         setDisplayValuesInRecord(table, fields, record, true);
       }
    }
 
 
 
    /*******************************************************************************
-    ** For a list of records, set their recordLabels and display values
+    ** For a single record, set its display values - public version of this.
     *******************************************************************************/
-   public static void setDisplayValuesInRecords(Map<String, QFieldMetaData> fields, List<QRecord> records)
+   public static void setDisplayValuesInRecord(QTableMetaData table, Map<String, QFieldMetaData> fields, QRecord record)
    {
-      if(records == null)
+      setDisplayValuesInRecord(table, fields, record, false);
-      {
-         return;
-      }
-
-      for(QRecord record : records)
-      {
-         setDisplayValuesInRecord(fields, record);
-      }
    }
 
 
-
    /*******************************************************************************
-    ** For a list of records, set their display values
+    ** For a single record, set its display values - where caller (meant to stay private)
+    ** can specify if they've already done fieldBehaviors (to avoid re-doing).
     *******************************************************************************/
-   public static void setDisplayValuesInRecord(Collection<QFieldMetaData> fields, QRecord record)
+   private static void setDisplayValuesInRecord(QTableMetaData table, Map<String, QFieldMetaData> fields, QRecord record, boolean alreadyAppliedFieldDisplayBehaviors)
    {
-      for(QFieldMetaData field : fields)
+      if(!alreadyAppliedFieldDisplayBehaviors)
       {
-         if(record.getDisplayValue(field.getName()) == null)
+         if(table != null)
          {
-            String formattedValue = formatValue(field, record.getValue(field.getName()));
+            ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, QContext.getQInstance(), table, List.of(record), null);
-            record.setDisplayValue(field.getName(), formattedValue);
          }
       }
-   }
 
-
-
-   /*******************************************************************************
-    ** For a list of records, set their display values
-    *******************************************************************************/
-   public static void setDisplayValuesInRecord(Map<String, QFieldMetaData> fields, QRecord record)
-   {
       for(Map.Entry<String, QFieldMetaData> entry : fields.entrySet())
       {
          String         fieldName = entry.getKey();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplier.java

@@ -27,6 +27,7 @@ import java.util.Set;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -44,7 +45,8 @@ public class ValueBehaviorApplier
    public enum Action
    {
       INSERT,
-      UPDATE
+      UPDATE,
+      FORMATTING
    }
 
 
@@ -63,7 +65,34 @@ public class ValueBehaviorApplier
       {
          for(FieldBehavior<?> fieldBehavior : CollectionUtils.nonNullCollection(field.getBehaviors()))
          {
-            fieldBehavior.apply(action, recordList, instance, table, field, behaviorsToOmit);
+            boolean applyBehavior = true;
+            if(behaviorsToOmit != null && behaviorsToOmit.contains(fieldBehavior))
+            {
+               /////////////////////////////////////////////////////////////////////////////////////////
+               // if we're given a set of behaviors to omit, and this behavior is in there, then skip //
+               /////////////////////////////////////////////////////////////////////////////////////////
+               applyBehavior = false;
+            }
+
+            if(Action.FORMATTING == action && !(fieldBehavior instanceof FieldDisplayBehavior<?>))
+            {
+               ////////////////////////////////////////////////////////////////////////////////////////////////
+               // for the formatting action, do not apply the behavior unless it is a field-display-behavior //
+               ////////////////////////////////////////////////////////////////////////////////////////////////
+               applyBehavior = false;
+            }
+            else if(Action.FORMATTING != action && fieldBehavior instanceof FieldDisplayBehavior<?>)
+            {
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               // for non-formatting actions, do not apply the behavior IF it is a field-display-behavior //
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               applyBehavior = false;
+            }
+
+            if(applyBehavior)
+            {
+               fieldBehavior.apply(action, recordList, instance, table, field);
+            }
          }
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -64,6 +64,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.ParentWidgetMetaD
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
@@ -810,7 +811,7 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void validateTableField(QInstance qInstance, String tableName, String fieldName, QTableMetaData table, QFieldMetaData field)
+   private <T extends FieldBehavior<T>> void validateTableField(QInstance qInstance, String tableName, String fieldName, QTableMetaData table, QFieldMetaData field)
    {
       assertCondition(Objects.equals(fieldName, field.getName()),
          "Inconsistent naming in table " + tableName + " for field " + fieldName + "/" + field.getName() + ".");
@@ -823,12 +824,32 @@ public class QInstanceValidator
 
       String prefix = "Field " + fieldName + " in table " + tableName + " ";
 
+      ///////////////////////////////////////////////////
+      // validate things we know about field behaviors //
+      ///////////////////////////////////////////////////
       ValueTooLongBehavior behavior = field.getBehaviorOrDefault(qInstance, ValueTooLongBehavior.class);
       if(behavior != null && !behavior.equals(ValueTooLongBehavior.PASS_THROUGH))
       {
          assertCondition(field.getMaxLength() != null, prefix + "specifies a ValueTooLongBehavior, but not a maxLength.");
       }
 
+      Set<Class<FieldBehavior<T>>> usedFieldBehaviorTypes = new HashSet<>();
+      if(field.getBehaviors() != null)
+      {
+         for(FieldBehavior<?> fieldBehavior : field.getBehaviors())
+         {
+            Class<FieldBehavior<T>> behaviorClass = (Class<FieldBehavior<T>>) fieldBehavior.getClass();
+
+            errors.addAll(fieldBehavior.validateBehaviorConfiguration(table, field));
+
+            if(!fieldBehavior.allowMultipleBehaviorsOfThisType())
+            {
+               assertCondition(!usedFieldBehaviorTypes.contains(behaviorClass), prefix + "has more than 1 fieldBehavior of type " + behaviorClass.getSimpleName() + ", which is not allowed for this type");
+            }
+            usedFieldBehaviorTypes.add(behaviorClass);
+         }
+      }
+
       if(field.getMaxLength() != null)
       {
          assertCondition(field.getMaxLength() > 0, prefix + "has an invalid maxLength (" + field.getMaxLength() + ") - must be greater than 0.");
@@ -1444,7 +1465,7 @@ public class QInstanceValidator
    private void validateScheduleMetaData(QScheduleMetaData schedule, QInstance qInstance, String prefix)
    {
       boolean isRepeat = schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null;
-      boolean isCron = StringUtils.hasContent(schedule.getCronExpression());
+      boolean isCron   = StringUtils.hasContent(schedule.getCronExpression());
       assertCondition(isRepeat || isCron, prefix + " either repeatMillis or repeatSeconds or cronExpression must be set");
       assertCondition(!(isRepeat && isCron), prefix + " both a repeat time and cronExpression may not be set");
 
@@ -1464,8 +1485,8 @@ public class QInstanceValidator
 
          if(assertCondition(StringUtils.hasContent(schedule.getCronTimeZoneId()), prefix + " a cron schedule must specify a cronTimeZoneId"))
          {
-            String[] availableIDs = TimeZone.getAvailableIDs();
+            String[]         availableIDs = TimeZone.getAvailableIDs();
-            Optional<String> first = Arrays.stream(availableIDs).filter(id -> id.equals(schedule.getCronTimeZoneId())).findFirst();
+            Optional<String> first        = Arrays.stream(availableIDs).filter(id -> id.equals(schedule.getCronTimeZoneId())).findFirst();
             assertCondition(first.isPresent(), prefix + " unrecognized cronTimeZoneId: " + schedule.getCronTimeZoneId());
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.EnumSet;
 import java.util.List;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
@@ -68,6 +69,24 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
    private boolean            includeAssociations       = false;
    private Collection<String> associationNamesToInclude = null;
 
+   private EnumSet<QueryHint> queryHints = EnumSet.noneOf(QueryHint.class);
+
+
+
+   /*******************************************************************************
+    ** Information about the query that an application (or qqq service) may know and
+    ** want to tell the backend, that can help influence how the backend processes
+    ** query.
+    **
+    ** For example, a query with potentially a large result set, for MySQL backend,
+    ** we may want to configure the result set to stream results rather than do its
+    ** default in-memory thing.  See RDBMSQueryAction for usage.
+    *******************************************************************************/
+   public enum QueryHint
+   {
+      POTENTIALLY_LARGE_NUMBER_OF_RESULTS
+   }
+
 
 
    /*******************************************************************************
@@ -569,4 +588,64 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for queryHints
+    *******************************************************************************/
+   public EnumSet<QueryHint> getQueryHints()
+   {
+      return (this.queryHints);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for queryHints
+    *******************************************************************************/
+   public void setQueryHints(EnumSet<QueryHint> queryHints)
+   {
+      this.queryHints = queryHints;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for queryHints
+    *******************************************************************************/
+   public QueryInput withQueryHints(EnumSet<QueryHint> queryHints)
+   {
+      this.queryHints = queryHints;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for queryHints
+    *******************************************************************************/
+   public QueryInput withQueryHint(QueryHint queryHint)
+   {
+      if(this.queryHints == null)
+      {
+         this.queryHints = EnumSet.noneOf(QueryHint.class);
+      }
+      this.queryHints.add(queryHint);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for queryHints
+    *******************************************************************************/
+   public QueryInput withoutQueryHint(QueryHint queryHint)
+   {
+      if(this.queryHints != null)
+      {
+         this.queryHints.remove(queryHint);
+      }
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java

@@ -39,7 +39,8 @@ public class ReplaceInput extends AbstractTableActionInput
    private UniqueKey           key;
    private List<QRecord>       records;
    private QQueryFilter        filter;
-   private boolean             performDeletes = true;
+   private boolean             performDeletes            = true;
+   private boolean             allowNullKeyValuesToEqual = false;
 
    private boolean omitDmlAudit = false;
 
@@ -239,4 +240,35 @@ public class ReplaceInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for allowNullKeyValuesToEqual
+    *******************************************************************************/
+   public boolean getAllowNullKeyValuesToEqual()
+   {
+      return (this.allowNullKeyValuesToEqual);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for allowNullKeyValuesToEqual
+    *******************************************************************************/
+   public void setAllowNullKeyValuesToEqual(boolean allowNullKeyValuesToEqual)
+   {
+      this.allowNullKeyValuesToEqual = allowNullKeyValuesToEqual;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for allowNullKeyValuesToEqual
+    *******************************************************************************/
+   public ReplaceInput withAllowNullKeyValuesToEqual(boolean allowNullKeyValuesToEqual)
+   {
+      this.allowNullKeyValuesToEqual = allowNullKeyValuesToEqual;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/AlertData.java

@@ -0,0 +1,139 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
+
+
+/*******************************************************************************
+ ** Model containing datastructure expected by frontend alert widget
+ **
+ *******************************************************************************/
+public class AlertData extends QWidgetData
+{
+   public enum AlertType
+   {
+      ERROR,
+      SUCCESS,
+      WARNING
+   }
+
+
+
+   private String    html;
+   private AlertType alertType;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public AlertData()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public AlertData(AlertType alertType, String html)
+   {
+      setHtml(html);
+      setAlertType(alertType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for type
+    **
+    *******************************************************************************/
+   public String getType()
+   {
+      return WidgetType.ALERT.getType();
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for html
+    **
+    *******************************************************************************/
+   public String getHtml()
+   {
+      return html;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for html
+    **
+    *******************************************************************************/
+   public void setHtml(String html)
+   {
+      this.html = html;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for html
+    **
+    *******************************************************************************/
+   public AlertData withHtml(String html)
+   {
+      this.html = html;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for alertType
+    *******************************************************************************/
+   public AlertType getAlertType()
+   {
+      return (this.alertType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for alertType
+    *******************************************************************************/
+   public void setAlertType(AlertType alertType)
+   {
+      this.alertType = alertType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for alertType
+    *******************************************************************************/
+   public AlertData withAlertType(AlertType alertType)
+   {
+      this.alertType = alertType;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/FieldValueListData.java

@@ -27,6 +27,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -147,7 +148,7 @@ public class FieldValueListData extends QWidgetData
          }
       }
 
-      QValueFormatter.setDisplayValuesInRecord(fields, record);
+      QValueFormatter.setDisplayValuesInRecord(null, fields.stream().collect(Collectors.toMap(f -> f.getName(), f -> f)), record);
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java

@@ -27,6 +27,7 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
  *******************************************************************************/
 public enum WidgetType
 {
+   ALERT("alert"),
    BAR_CHART("barChart"),
    CHART("chart"),
    CHILD_RECORD_LIST("childRecordList"),

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java

@@ -150,7 +150,7 @@ public class MetaDataProducerHelper
             }
             catch(Exception e)
             {
-               LOG.warn("error executing metaDataProducer", logPair("producer", producer.getClass().getSimpleName()), e);
+               LOG.warn("error executing metaDataProducer", e, logPair("producer", producer.getClass().getSimpleName()));
             }
          }
          else

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownData.java

@@ -28,6 +28,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.dashboard;
  *******************************************************************************/
 public class WidgetDropdownData
 {
+   private String  name;
    private String  possibleValueSourceName;
    private String  foreignKeyFieldName;
    private String  label;
@@ -44,6 +45,9 @@ public class WidgetDropdownData
    ////////////////////////////////////////////////////////////////////////////////////////////////
    private String labelForNullValue;
 
+   private WidgetDropdownType type = WidgetDropdownType.POSSIBLE_VALUE_SOURCE;
+
+
 
    /*******************************************************************************
     ** Getter for possibleValueSourceName
@@ -366,4 +370,65 @@ public class WidgetDropdownData
    }
 
 
+
+   /*******************************************************************************
+    ** Getter for type
+    *******************************************************************************/
+   public WidgetDropdownType getType()
+   {
+      return (this.type);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for type
+    *******************************************************************************/
+   public void setType(WidgetDropdownType type)
+   {
+      this.type = type;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for type
+    *******************************************************************************/
+   public WidgetDropdownData withType(WidgetDropdownType type)
+   {
+      this.type = type;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for name
+    *******************************************************************************/
+   public String getName()
+   {
+      return (this.name);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for name
+    *******************************************************************************/
+   public void setName(String name)
+   {
+      this.name = name;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for name
+    *******************************************************************************/
+   public WidgetDropdownData withName(String name)
+   {
+      this.name = name;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownType.java

@@ -0,0 +1,33 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.dashboard;
+
+
+/*******************************************************************************
+ ** Possible types for widget dropdowns
+ **
+ *******************************************************************************/
+public enum WidgetDropdownType
+{
+   POSSIBLE_VALUE_SOURCE,
+   DATE_PICKER
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehavior.java

@@ -0,0 +1,331 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.fields;
+
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Field Display Behavior class for customizing the display values used
+ ** in date-time fields
+ *******************************************************************************/
+public class DateTimeDisplayValueBehavior implements FieldDisplayBehavior<DateTimeDisplayValueBehavior>
+{
+   private static final QLogger LOG = QLogger.getLogger(DateTimeDisplayValueBehavior.class);
+
+   private String zoneIdFromFieldName;
+   private String fallbackZoneId;
+
+   private String defaultZoneId;
+
+   private static DateTimeDisplayValueBehavior NOOP = new DateTimeDisplayValueBehavior();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public DateTimeDisplayValueBehavior getDefault()
+   {
+      return NOOP;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
+   {
+      if(StringUtils.hasContent(defaultZoneId))
+      {
+         applyDefaultZoneId(recordList, table, field);
+      }
+      else if(StringUtils.hasContent(zoneIdFromFieldName))
+      {
+         applyZoneIdFromFieldName(recordList, table, field);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void applyDefaultZoneId(List<QRecord> recordList, QTableMetaData table, QFieldMetaData field)
+   {
+      for(QRecord record : CollectionUtils.nonNullList(recordList))
+      {
+         try
+         {
+            Instant       instant       = record.getValueInstant(field.getName());
+            ZonedDateTime zonedDateTime = instant.atZone(ZoneId.of(defaultZoneId));
+            record.setDisplayValue(field.getName(), QValueFormatter.formatDateTimeWithZone(zonedDateTime));
+         }
+         catch(Exception e)
+         {
+            LOG.info("Error applying defaultZoneId DateTimeDisplayValueBehavior", logPair("table", table.getName()), logPair("field", field.getName()), logPair("id", record.getValue(table.getPrimaryKeyField())));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void applyZoneIdFromFieldName(List<QRecord> recordList, QTableMetaData table, QFieldMetaData field)
+   {
+      for(QRecord record : CollectionUtils.nonNullList(recordList))
+      {
+         try
+         {
+            Instant instant    = record.getValueInstant(field.getName());
+            String  zoneString = record.getValueString(zoneIdFromFieldName);
+
+            ZoneId zoneId;
+            try
+            {
+               zoneId = ZoneId.of(zoneString);
+            }
+            catch(Exception e)
+            {
+               ////////////////////////////////////////////////////////////////////////////////////////////////
+               // if the zone string from the other field isn't valid, and we have a fallback, try to use it //
+               ////////////////////////////////////////////////////////////////////////////////////////////////
+               if(StringUtils.hasContent(fallbackZoneId))
+               {
+                  zoneId = ZoneId.of(fallbackZoneId);
+               }
+               else
+               {
+                  throw (e);
+               }
+            }
+
+            ZonedDateTime zonedDateTime = instant.atZone(zoneId);
+            record.setDisplayValue(field.getName(), QValueFormatter.formatDateTimeWithZone(zonedDateTime));
+         }
+         catch(Exception e)
+         {
+            LOG.info("Error applying zoneIdFromFieldName DateTimeDisplayValueBehavior", e, logPair("table", table.getName()), logPair("field", field.getName()), logPair("id", record.getValue(table.getPrimaryKeyField())));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<String> validateBehaviorConfiguration(QTableMetaData tableMetaData, QFieldMetaData fieldMetaData)
+   {
+      List<String> errors      = new ArrayList<>();
+      String       errorSuffix = " field [" + fieldMetaData.getName() + "] in table [" + tableMetaData.getName() + "]";
+
+      if(!QFieldType.DATE_TIME.equals(fieldMetaData.getType()))
+      {
+         errors.add("A DateTimeDisplayValueBehavior was a applied to a non-DATE_TIME" + errorSuffix);
+      }
+
+      //////////////////////////////////////////////////
+      // validate rules if zoneIdFromFieldName is set //
+      //////////////////////////////////////////////////
+      if(StringUtils.hasContent(zoneIdFromFieldName))
+      {
+         if(StringUtils.hasContent(defaultZoneId))
+         {
+            errors.add("You may not specify both zoneIdFromFieldName and defaultZoneId in DateTimeDisplayValueBehavior on" + errorSuffix);
+         }
+
+         if(!tableMetaData.getFields().containsKey(zoneIdFromFieldName))
+         {
+            errors.add("Unrecognized field name [" + zoneIdFromFieldName + "] for [zoneIdFromFieldName] in DateTimeDisplayValueBehavior on" + errorSuffix);
+         }
+         else
+         {
+            QFieldMetaData zoneIdField = tableMetaData.getFields().get(zoneIdFromFieldName);
+            if(!QFieldType.STRING.equals(zoneIdField.getType()))
+            {
+               errors.add("A non-STRING type [" + zoneIdField.getType() + "] was specified as the zoneIdFromFieldName field [" + zoneIdFromFieldName + "] in DateTimeDisplayValueBehavior on" + errorSuffix);
+            }
+         }
+      }
+
+      ////////////////////////////////////////////
+      // validate rules if defaultZoneId is set //
+      ////////////////////////////////////////////
+      if(StringUtils.hasContent(defaultZoneId))
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         // would check that you didn't specify from zoneIdFromFieldName - but that's covered above //
+         /////////////////////////////////////////////////////////////////////////////////////////////
+
+         if(StringUtils.hasContent(fallbackZoneId))
+         {
+            errors.add("You may not specify both defaultZoneId and fallbackZoneId in DateTimeDisplayValueBehavior on" + errorSuffix);
+         }
+
+         try
+         {
+            ZoneId.of(defaultZoneId);
+         }
+         catch(Exception e)
+         {
+            errors.add("Invalid ZoneId [" + defaultZoneId + "] for [defaultZoneId] in DateTimeDisplayValueBehavior on" + errorSuffix + "; " + e.getMessage());
+         }
+      }
+
+      /////////////////////////////////////////////
+      // validate rules if fallbackZoneId is set //
+      /////////////////////////////////////////////
+      if(StringUtils.hasContent(fallbackZoneId))
+      {
+         if(!StringUtils.hasContent(zoneIdFromFieldName))
+         {
+            errors.add("You may only set fallbackZoneId if using zoneIdFromFieldName in DateTimeDisplayValueBehavior on" + errorSuffix);
+         }
+
+         try
+         {
+            ZoneId.of(fallbackZoneId);
+         }
+         catch(Exception e)
+         {
+            errors.add("Invalid ZoneId [" + fallbackZoneId + "] for [fallbackZoneId] in DateTimeDisplayValueBehavior on" + errorSuffix + "; " + e.getMessage());
+         }
+      }
+
+      return (errors);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for zoneIdFromFieldName
+    *******************************************************************************/
+   public String getZoneIdFromFieldName()
+   {
+      return (this.zoneIdFromFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for zoneIdFromFieldName
+    *******************************************************************************/
+   public void setZoneIdFromFieldName(String zoneIdFromFieldName)
+   {
+      this.zoneIdFromFieldName = zoneIdFromFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for zoneIdFromFieldName
+    *******************************************************************************/
+   public DateTimeDisplayValueBehavior withZoneIdFromFieldName(String zoneIdFromFieldName)
+   {
+      this.zoneIdFromFieldName = zoneIdFromFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for defaultZoneId
+    *******************************************************************************/
+   public String getDefaultZoneId()
+   {
+      return (this.defaultZoneId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for defaultZoneId
+    *******************************************************************************/
+   public void setDefaultZoneId(String defaultZoneId)
+   {
+      this.defaultZoneId = defaultZoneId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for defaultZoneId
+    *******************************************************************************/
+   public DateTimeDisplayValueBehavior withDefaultZoneId(String defaultZoneId)
+   {
+      this.defaultZoneId = defaultZoneId;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for fallbackZoneId
+    *******************************************************************************/
+   public String getFallbackZoneId()
+   {
+      return (this.fallbackZoneId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fallbackZoneId
+    *******************************************************************************/
+   public void setFallbackZoneId(String fallbackZoneId)
+   {
+      this.fallbackZoneId = fallbackZoneId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fallbackZoneId
+    *******************************************************************************/
+   public DateTimeDisplayValueBehavior withFallbackZoneId(String fallbackZoneId)
+   {
+      this.fallbackZoneId = fallbackZoneId;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehavior.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -26,7 +26,6 @@ import java.io.Serializable;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.util.List;
-import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -66,16 +65,12 @@ public enum DynamicDefaultValueBehavior implements FieldBehavior<DynamicDefaultV
     **
     *******************************************************************************/
    @Override
-   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field, Set<FieldBehavior<?>> behaviorsToOmit)
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
    {
       if(this.equals(NONE))
       {
          return;
       }
-      if(behaviorsToOmit != null && behaviorsToOmit.contains(this))
-      {
-         return;
-      }
 
       switch(this)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldBehavior.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -22,8 +22,9 @@
 package com.kingsrook.qqq.backend.core.model.metadata.fields;
 
 
+import java.util.Collections;
 import java.util.List;
-import java.util.Set;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -34,8 +35,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
  ** Interface for (expected to be?) enums which define behaviors that get applied
  ** to fields.
  **
- ** At the present, these behaviors get applied before a field is stored (insert
+ ** Some of these behaviors get applied before a field is stored (insert
- ** or update), through the ValueBehaviorApplier class.
+ ** or update), through the ValueBehaviorApplier class.  Others can be used to
+ ** do more advanced display formatting than the displayFormat string alone can
+ ** do (see QValueFormatter).
  **
  *******************************************************************************/
 public interface FieldBehavior<T extends FieldBehavior<T>>
@@ -45,12 +48,13 @@ public interface FieldBehavior<T extends FieldBehavior<T>>
     ** In case a behavior of this type wasn't set on the field, what should the
     ** default of this type be?
     *******************************************************************************/
+   @JsonIgnore
    T getDefault();
 
    /*******************************************************************************
     ** Apply this behavior to a list of records
     *******************************************************************************/
-   void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field, Set<FieldBehavior<?>> behaviorsToOmit);
+   void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field);
 
    /*******************************************************************************
     ** control if multiple behaviors of this type should be allowed together on a field.
@@ -60,4 +64,14 @@ public interface FieldBehavior<T extends FieldBehavior<T>>
       return (false);
    }
 
+   /*******************************************************************************
+    ** allow this behavior to be validated during QInstance validation.
+    **
+    ** return a list of validation errors, if there are any.
+    *******************************************************************************/
+   default List<String> validateBehaviorConfiguration(QTableMetaData tableMetaData, QFieldMetaData fieldMetaData)
+   {
+      return (Collections.emptyList());
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldDisplayBehavior.java

@@ -0,0 +1,31 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.fields;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public interface FieldDisplayBehavior<T extends FieldDisplayBehavior<T>> extends FieldBehavior<T>
+{
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java

@@ -716,6 +716,17 @@ public class QFieldMetaData implements Cloneable
       {
          return (behaviorType.getEnumConstants()[0].getDefault());
       }
+      else
+      {
+         try
+         {
+            return (behaviorType.getConstructor().newInstance().getDefault());
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Error getting default behaviorType for [" + behaviorType.getSimpleName() + "]", e);
+         }
+      }
 
       return (null);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/ValueTooLongBehavior.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields;
 
 
 import java.util.List;
-import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -66,16 +65,12 @@ public enum ValueTooLongBehavior implements FieldBehavior<ValueTooLongBehavior>
     **
     *******************************************************************************/
    @Override
-   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field, Set<FieldBehavior<?>> behaviorsToOmit)
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
    {
       if(this.equals(PASS_THROUGH))
       {
          return;
       }
-      if(behaviorsToOmit != null && behaviorsToOmit.contains(this))
-      {
-         return;
-      }
 
       String fieldName = field.getName();
       if(!QFieldType.STRING.equals(field.getType()))

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleDispatcher.java

@@ -27,6 +27,7 @@ import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -89,7 +90,7 @@ public class QBackendModuleDispatcher
          }
          catch(Exception e)
          {
-            LOG.debug("Backend module [{}] could not be loaded: {}", moduleClassName, e.getMessage());
+            LOG.debug("Backend module could not be loaded", e, logPair("moduleClassName", moduleClassName));
          }
       }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java

@@ -34,6 +34,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.DateTimeGroupBy;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
 import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
@@ -252,7 +253,7 @@ public class ColumnStatsStep implements BackendStep
 
          QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator();
          qPossibleValueTranslator.translatePossibleValuesInRecords(table, valueCounts, queryJoin == null ? null : List.of(queryJoin), null);
-         QValueFormatter.setDisplayValuesInRecords(Map.of(fieldName, field, "count", countField), valueCounts);
+         QValueFormatter.setDisplayValuesInRecords(table, Map.of(fieldName, field, "count", countField), valueCounts);
 
          runBackendStepOutput.addValue("valueCounts", valueCounts);
 
@@ -442,13 +443,13 @@ public class ColumnStatsStep implements BackendStep
             }
 
             QFieldMetaData percentField = new QFieldMetaData("percent", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.PERCENT_POINT2).withLabel("Percent");
-            QValueFormatter.setDisplayValuesInRecords(Map.of(fieldName, field, "percent", percentField), valueCounts);
+            QValueFormatter.setDisplayValuesInRecords(table, Map.of(fieldName, field, "percent", percentField), valueCounts);
          }
 
          QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(null);
          fields.forEach(qInstanceEnricher::enrichField);
 
-         QValueFormatter.setDisplayValuesInRecord(fields, statsRecord);
+         QValueFormatter.setDisplayValuesInRecord(table, fields.stream().collect(Collectors.toMap(f -> f.getName(), f -> f)), statsRecord);
 
          runBackendStepOutput.addValue("statsFields", fields);
          runBackendStepOutput.addValue("statsRecord", statsRecord);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java

@@ -113,6 +113,10 @@ public class ExtractViaQueryStep extends AbstractExtractStep
       {
          queryInput.setShouldFetchHeavyFields(true);
       }
+      if(runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_INCLUDE_ASSOCIATIONS))
+      {
+         queryInput.setIncludeAssociations(true);
+      }
 
       customizeInputPreQuery(queryInput);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java

@@ -84,6 +84,7 @@ public class StreamedETLWithFrontendProcess
    public static final String FIELD_RECORD_COUNT         = "recordCount"; // Integer
    public static final String FIELD_DEFAULT_QUERY_FILTER = "defaultQueryFilter"; // QQueryFilter or String (json, of q QQueryFilter)
    public static final String FIELD_FETCH_HEAVY_FIELDS   = "fetchHeavyFields"; // Boolean
+   public static final String FIELD_INCLUDE_ASSOCIATIONS = "includeAssociations"; // Boolean
 
    public static final String FIELD_SUPPORTS_FULL_VALIDATION = "supportsFullValidation"; // Boolean
    public static final String FIELD_DO_FULL_VALIDATION       = "doFullValidation"; // Boolean
@@ -145,6 +146,7 @@ public class StreamedETLWithFrontendProcess
          .withCode(new QCodeReference(StreamedETLPreviewStep.class))
          .withInputData(new QFunctionInputMetaData()
             .withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_SOURCE_TABLE)))
+            .withField(new QFieldMetaData(FIELD_INCLUDE_ASSOCIATIONS, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_INCLUDE_ASSOCIATIONS, false)))
             .withField(new QFieldMetaData(FIELD_FETCH_HEAVY_FIELDS, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_FETCH_HEAVY_FIELDS, false)))
             .withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DESTINATION_TABLE)))
             .withField(new QFieldMetaData(FIELD_SUPPORTS_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_SUPPORTS_FULL_VALIDATION, true)))

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorExtractStep.java

@@ -26,6 +26,7 @@ import java.time.Instant;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
@@ -57,4 +58,15 @@ public class GarbageCollectorExtractStep extends ExtractViaQueryStep
       return super.getQueryFilter(runBackendStepInput);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   protected void customizeInputPreQuery(QueryInput queryInput)
+   {
+      queryInput.withQueryHint(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mock/MockBackendStep.java

@@ -56,7 +56,7 @@ public class MockBackendStep implements BackendStep
 
       runBackendStepInput.getRecords().forEach(r ->
       {
-         LOG.info("We are mocking {}: {}", r.getValueString("firstName"), r.getValue(FIELD_MOCK_VALUE));
+         LOG.info("We are mocking " + r.getValueString("firstName") + ": " + r.getValue(FIELD_MOCK_VALUE));
          r.setValue(FIELD_MOCK_VALUE, "Ha ha!");
          r.setValue("greetingMessage", runBackendStepInput.getValueString(FIELD_GREETING_PREFIX) + " " + r.getValueString("firstName") + " " + runBackendStepInput.getValueString(FIELD_GREETING_SUFFIX));
       });

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/AbstractStateKey.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.state;
 
 
 import java.io.Serializable;
+import java.time.Instant;
 
 
 /*******************************************************************************
@@ -57,4 +58,10 @@ public abstract class AbstractStateKey implements Serializable
    @Override
    public abstract String toString();
 
+   /*******************************************************************************
+    ** Require all state keys to implement the getStartTime method
+    *
+    *******************************************************************************/
+   public abstract Instant getStartTime();
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java

@@ -23,9 +23,16 @@ package com.kingsrook.qqq.backend.core.state;
 
 
 import java.io.Serializable;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -33,10 +40,16 @@ import java.util.Optional;
  *******************************************************************************/
 public class InMemoryStateProvider implements StateProviderInterface
 {
+   private static final QLogger LOG = QLogger.getLogger(InMemoryStateProvider.class);
+
    private static InMemoryStateProvider instance;
 
    private final Map<AbstractStateKey, Object> map;
 
+   private static int jobPeriodSeconds = 60 * 60; // 1 hour
+   private static int cleanHours       = 6;
+   private static int jobInitialDelay  = 60 * 60 * cleanHours;
+
 
 
    /*******************************************************************************
@@ -45,6 +58,41 @@ public class InMemoryStateProvider implements StateProviderInterface
    private InMemoryStateProvider()
    {
       this.map = new HashMap<>();
+
+      ///////////////////////////////////////////////////////////
+      // Start a single thread executor to handle the cleaning //
+      ///////////////////////////////////////////////////////////
+      ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
+      executorService.scheduleAtFixedRate(new InMemoryStateProvider.InMemoryStateProviderCleanJob(), jobInitialDelay, jobPeriodSeconds, TimeUnit.SECONDS);
+   }
+
+
+
+   /*******************************************************************************
+    ** Runnable that gets scheduled to periodically clean the InMemoryStateProvider
+    *******************************************************************************/
+   private static class InMemoryStateProviderCleanJob implements Runnable
+   {
+      private static final QLogger LOG = QLogger.getLogger(InMemoryStateProvider.InMemoryStateProviderCleanJob.class);
+
+
+
+      /*******************************************************************************
+       ** run
+       *******************************************************************************/
+      @Override
+      public void run()
+      {
+         try
+         {
+            Instant cleanTime = Instant.now().minus(cleanHours, ChronoUnit.HOURS);
+            getInstance().clean(cleanTime);
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Error cleaning InMemoryStateProvider entries.", e);
+         }
+      }
    }
 
 
@@ -101,4 +149,24 @@ public class InMemoryStateProvider implements StateProviderInterface
       map.remove(key);
    }
 
+
+
+   /*******************************************************************************
+    ** Clean entries that started before the given Instant
+    *
+    *******************************************************************************/
+   @Override
+   public void clean(Instant cleanBeforeInstant)
+   {
+      long    jobStartTime = System.currentTimeMillis();
+      Integer beforeSize   = map.size();
+      LOG.info("Starting clean for InMemoryStateProvider.", logPair("beforeSize", beforeSize));
+
+      map.entrySet().removeIf(e -> e.getKey().getStartTime().isBefore(cleanBeforeInstant));
+
+      Integer afterSize = map.size();
+      long    endTime   = System.currentTimeMillis();
+      LOG.info("Completed clean for InMemoryStateProvider.", logPair("beforeSize", beforeSize), logPair("afterSize", afterSize), logPair("amountCleaned", (beforeSize - afterSize)), logPair("runTimeMillis", (endTime - jobStartTime)));
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/SimpleStateKey.java

@@ -22,6 +22,9 @@
 package com.kingsrook.qqq.backend.core.state;
 
 
+import java.time.Instant;
+
+
 /*******************************************************************************
  **
  *******************************************************************************/
@@ -93,4 +96,17 @@ public class SimpleStateKey<T> extends AbstractStateKey
    {
       return key.hashCode();
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for startTime
+    *******************************************************************************/
+   public Instant getStartTime()
+   {
+      //////////////////////////////////////////
+      // For now these will never get cleaned //
+      //////////////////////////////////////////
+      return (Instant.now());
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/StateProviderInterface.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.state;
 
 
 import java.io.Serializable;
+import java.time.Instant;
 import java.util.Optional;
 
 
@@ -58,4 +59,8 @@ public interface StateProviderInterface
     *******************************************************************************/
    void remove(AbstractStateKey key);
 
+   /*******************************************************************************
+    ** Clean entries that started before the given Instant
+    *******************************************************************************/
+   void clean(Instant startTime);
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java

@@ -26,6 +26,7 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Serializable;
+import java.time.Instant;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
@@ -126,6 +127,19 @@ public class TempFileStateProvider implements StateProviderInterface
 
 
 
+   /*******************************************************************************
+    ** Clean entries that started before the given Instant
+    *******************************************************************************/
+   @Override
+   public void clean(Instant startTime)
+   {
+      ////////////////////////////////
+      // Not supported at this time //
+      ////////////////////////////////
+   }
+
+
+
    /*******************************************************************************
     ** Get the file referenced by a key
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/UUIDAndTypeStateKey.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.state;
 
 
 import java.io.Serializable;
+import java.time.Instant;
 import java.util.Objects;
 import java.util.UUID;
 
@@ -34,6 +35,7 @@ public class UUIDAndTypeStateKey extends AbstractStateKey implements Serializabl
 {
    private final UUID      uuid;
    private final StateType stateType;
+   private final Instant   startTime;
 
 
 
@@ -43,7 +45,7 @@ public class UUIDAndTypeStateKey extends AbstractStateKey implements Serializabl
     *******************************************************************************/
    public UUIDAndTypeStateKey(StateType stateType)
    {
-      this(UUID.randomUUID(), stateType);
+      this(UUID.randomUUID(), stateType, Instant.now());
    }
 
 
@@ -53,9 +55,21 @@ public class UUIDAndTypeStateKey extends AbstractStateKey implements Serializabl
     **
     *******************************************************************************/
    public UUIDAndTypeStateKey(UUID uuid, StateType stateType)
+   {
+      this(uuid, stateType, Instant.now());
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor where user can supply the UUID.
+    **
+    *******************************************************************************/
+   public UUIDAndTypeStateKey(UUID uuid, StateType stateType, Instant startTime)
    {
       this.uuid = uuid;
       this.stateType = stateType;
+      this.startTime = startTime;
    }
 
 
@@ -133,4 +147,15 @@ public class UUIDAndTypeStateKey extends AbstractStateKey implements Serializabl
    {
       return "{uuid=" + uuid + ", stateType=" + stateType + '}';
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for startTime
+    *******************************************************************************/
+   public Instant getStartTime()
+   {
+      return (this.startTime);
+   }
+
 }

qqq-backend-core/src/main/resources/log4j2.xml

@@ -31,6 +31,8 @@
       </Logger>
       <Logger name="org.quartz" level="INFO">
       </Logger>
+      <Logger name="liquibase" level="INFO">
+      </Logger>
       <Root level="all">
          <AppenderRef ref="SystemOutAppender"/>
          <AppenderRef ref="SyslogAppender"/>

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java

@@ -43,7 +43,7 @@ import static org.junit.jupiter.api.Assertions.assertNull;
 
 
 /*******************************************************************************
- ** Unit test for ReplaceAction 
+ ** Unit test for ReplaceAction
  *******************************************************************************/
 class ReplaceActionTest extends BaseTest
 {
@@ -157,6 +157,134 @@ class ReplaceActionTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTwoKeysWithNullsNotMatchingAllowingDelete() throws QException
+   {
+      String tableName = TestUtils.TABLE_NAME_TWO_KEYS;
+
+      ////////////////////////////////
+      // start with these 2 records //
+      ////////////////////////////////
+      new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(
+         new QRecord().withValue("key1", 1).withValue("key2", 2),
+         new QRecord().withValue("key1", 3)
+      )));
+
+      ////////////////////////////////////////////////////
+      // now do a replace action that just updates them //
+      ////////////////////////////////////////////////////
+      List<QRecord> newThings = List.of(
+         new QRecord().withValue("key1", 1).withValue("key2", 2),
+         new QRecord().withValue("key1", 3)
+      );
+
+      //////////////////////////////
+      // replace allowing deletes //
+      //////////////////////////////
+      ReplaceInput replaceInput = new ReplaceInput();
+      replaceInput.setTableName(tableName);
+      replaceInput.setKey(new UniqueKey("key1", "key2"));
+      replaceInput.setOmitDmlAudit(true);
+      replaceInput.setRecords(newThings);
+      replaceInput.setFilter(null);
+      ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput);
+
+      assertEquals(1, replaceOutput.getInsertOutput().getRecords().size());
+      assertEquals(1, replaceOutput.getUpdateOutput().getRecords().size());
+      assertEquals(1, replaceOutput.getDeleteOutput().getDeletedRecordCount());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTwoKeysWithNullsNotMatchingNotAllowingDelete() throws QException
+   {
+      String tableName = TestUtils.TABLE_NAME_TWO_KEYS;
+
+      ////////////////////////////////
+      // start with these 2 records //
+      ////////////////////////////////
+      new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(
+         new QRecord().withValue("key1", 1).withValue("key2", 2),
+         new QRecord().withValue("key1", 3)
+      )));
+
+      ////////////////////////////////////////////////////
+      // now do a replace action that just updates them //
+      ////////////////////////////////////////////////////
+      List<QRecord> newThings = List.of(
+         new QRecord().withValue("key1", 1).withValue("key2", 2),
+         new QRecord().withValue("key1", 3)
+      );
+
+      /////////////////////////////////
+      // replace disallowing deletes //
+      /////////////////////////////////
+      ReplaceInput replaceInput = new ReplaceInput();
+      replaceInput.setTableName(tableName);
+      replaceInput.setKey(new UniqueKey("key1", "key2"));
+      replaceInput.setOmitDmlAudit(true);
+      replaceInput.setRecords(newThings);
+      replaceInput.setFilter(null);
+      replaceInput.setPerformDeletes(false);
+      ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput);
+
+      assertEquals(1, replaceOutput.getInsertOutput().getRecords().size());
+      assertEquals(1, replaceOutput.getUpdateOutput().getRecords().size());
+      assertNull(replaceOutput.getDeleteOutput());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTwoKeysWithNullMatching() throws QException
+   {
+      String tableName = TestUtils.TABLE_NAME_TWO_KEYS;
+
+      ////////////////////////////////
+      // start with these 2 records //
+      ////////////////////////////////
+      new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(
+         new QRecord().withValue("key1", 1).withValue("key2", 2),
+         new QRecord().withValue("key1", 3)
+      )));
+
+      ////////////////////////////////////////////////////
+      // now do a replace action that just updates them //
+      ////////////////////////////////////////////////////
+      List<QRecord> newThings = List.of(
+         new QRecord().withValue("key1", 1).withValue("key2", 2),
+         new QRecord().withValue("key1", 3)
+      );
+
+      ///////////////////////////////////////////////
+      // replace treating null key values as equal //
+      ///////////////////////////////////////////////
+      ReplaceInput replaceInput = new ReplaceInput();
+      replaceInput.setTableName(tableName);
+      replaceInput.setKey(new UniqueKey("key1", "key2"));
+      replaceInput.setOmitDmlAudit(true);
+      replaceInput.setRecords(newThings);
+      replaceInput.setFilter(null);
+      replaceInput.setAllowNullKeyValuesToEqual(true);
+      ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput);
+
+      assertEquals(0, replaceOutput.getInsertOutput().getRecords().size());
+      assertEquals(2, replaceOutput.getUpdateOutput().getRecords().size());
+      assertEquals(0, replaceOutput.getDeleteOutput().getDeletedRecordCount());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -297,4 +425,4 @@ class ReplaceActionTest extends BaseTest
       return new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withUniqueKey(Map.of("firstName", firstName, "lastName", lastName))).getValueInteger("noOfShoes");
    }
 
-}
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.values;
 
 
 import java.math.BigDecimal;
+import java.time.Instant;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.LocalTime;
@@ -32,10 +33,13 @@ import java.time.ZonedDateTime;
 import java.util.Collections;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DateTimeDisplayValueBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
@@ -210,4 +214,23 @@ class QValueFormatterTest extends BaseTest
       assertEquals("2023-02-01 07:15:47 PM CST", QValueFormatter.formatDateTimeWithZone(ZonedDateTime.of(LocalDateTime.of(2023, Month.FEBRUARY, 1, 19, 15, 47), ZoneId.of("US/Central"))));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFieldDisplayBehaviors()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+
+      table.withField(new QFieldMetaData("timeZone", QFieldType.STRING));
+      table.getField("createDate").withBehavior(new DateTimeDisplayValueBehavior().withZoneIdFromFieldName("timeZone"));
+
+      QRecord record = new QRecord().withValue("createDate", Instant.parse("2024-04-04T19:12:00Z")).withValue("timeZone", "America/Chicago");
+      QValueFormatter.setDisplayValuesInRecords(table, List.of(record));
+      assertEquals("2024-04-04 02:12:00 PM CDT", record.getDisplayValue("createDate"));
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplierTest.java

@@ -29,9 +29,12 @@ import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -140,6 +143,36 @@ class ValueBehaviorApplierTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testApplyFormattingBehaviors()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+
+      table.getField("firstName").withBehavior(ToUpperCaseBehavior.getInstance());
+      table.getField("lastName").withBehavior(ToUpperCaseBehavior.NOOP);
+      table.getField("ssn").withBehavior(ValueTooLongBehavior.TRUNCATE).withMaxLength(1);
+
+      QRecord record = new QRecord().withValue("firstName", "Homer").withValue("lastName", "Simpson").withValue("ssn", "0123456789");
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, qInstance, table, List.of(record), null);
+      
+      assertEquals("HOMER", record.getDisplayValue("firstName"));
+      assertNull(record.getDisplayValue("lastName")); // noop will literally do nothing, not even pass value through.
+      assertEquals("0123456789", record.getValueString("ssn")); // formatting action should not run the too-long truncate behavior
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // now put to-upper-case behavior on lastName, but run INSERT actions - and make sure it doesn't get applied. //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      table.getField("lastName").withBehavior(ToUpperCaseBehavior.getInstance());
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record), null);
+      assertNull(record.getDisplayValue("lastName"));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -153,4 +186,73 @@ class ValueBehaviorApplierTest extends BaseTest
       return (recordOpt.get());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class ToUpperCaseBehavior implements FieldDisplayBehavior<ToUpperCaseBehavior>
+   {
+      private final boolean enabled;
+
+      private static ToUpperCaseBehavior NOOP     = new ToUpperCaseBehavior(false);
+      private static ToUpperCaseBehavior instance = new ToUpperCaseBehavior(true);
+
+
+
+      /*******************************************************************************
+       ** Constructor
+       **
+       *******************************************************************************/
+      private ToUpperCaseBehavior(boolean enabled)
+      {
+         this.enabled = enabled;
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public ToUpperCaseBehavior getDefault()
+      {
+         return (NOOP);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public static ToUpperCaseBehavior getInstance()
+      {
+         return (instance);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
+      {
+         if(!enabled)
+         {
+            return;
+         }
+
+         for(QRecord record : CollectionUtils.nonNullList(recordList))
+         {
+            String displayValue = record.getValueString(field.getName());
+            if(displayValue != null)
+            {
+               displayValue = displayValue.toUpperCase();
+            }
+
+            record.setDisplayValue(field.getName(), displayValue);
+         }
+      }
+   }
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -26,6 +26,8 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Set;
+import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -56,6 +58,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DateTimeDisplayValueBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
@@ -1739,6 +1742,26 @@ public class QInstanceValidatorTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFieldBehaviors()
+   {
+      BiFunction<QInstance, String, QFieldMetaData> fieldExtractor = (QInstance qInstance, String fieldName) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField(fieldName);
+      assertValidationFailureReasons((qInstance -> fieldExtractor.apply(qInstance, "firstName").withBehaviors(Set.of(ValueTooLongBehavior.ERROR, ValueTooLongBehavior.TRUNCATE)).withMaxLength(1)),
+         "more than 1 fieldBehavior of type ValueTooLongBehavior, which is not allowed");
+
+      ///////////////////////////////////////////////////////////////////////////
+      // make sure a custom validation method in a field behavior gets applied //
+      // more tests for this particular behavior are in its own test class     //
+      ///////////////////////////////////////////////////////////////////////////
+      assertValidationFailureReasons((qInstance -> fieldExtractor.apply(qInstance, "firstName").withBehavior(new DateTimeDisplayValueBehavior())),
+         "DateTimeDisplayValueBehavior was a applied to a non-DATE_TIME field");
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehaviorTest.java

@@ -0,0 +1,169 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.fields;
+
+
+import java.time.Instant;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for DateTimeDisplayValueBehavior 
+ *******************************************************************************/
+class DateTimeDisplayValueBehaviorTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testZoneIdFromFieldName()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+
+      table.withField(new QFieldMetaData("timeZone", QFieldType.STRING));
+      table.getField("createDate").withBehavior(new DateTimeDisplayValueBehavior().withZoneIdFromFieldName("timeZone"));
+
+      QRecord record = new QRecord().withValue("createDate", Instant.parse("2024-04-04T19:12:00Z")).withValue("timeZone", "America/Chicago");
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, qInstance, table, List.of(record), null);
+      assertEquals("2024-04-04 02:12:00 PM CDT", record.getDisplayValue("createDate"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testZoneIdFromFieldNameWithFallback()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+
+      table.withField(new QFieldMetaData("timeZone", QFieldType.STRING));
+      table.getField("createDate").withBehavior(new DateTimeDisplayValueBehavior().withZoneIdFromFieldName("timeZone").withFallbackZoneId("America/Denver"));
+
+      QRecord record = new QRecord().withValue("createDate", Instant.parse("2024-04-04T19:12:00Z")).withValue("timeZone", "whodis");
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, qInstance, table, List.of(record), null);
+      assertEquals("2024-04-04 01:12:00 PM MDT", record.getDisplayValue("createDate"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testDefaultZoneId()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+
+      table.withField(new QFieldMetaData("timeZone", QFieldType.STRING));
+      table.getField("createDate").withBehavior(new DateTimeDisplayValueBehavior().withDefaultZoneId("America/Los_Angeles"));
+
+      QRecord record = new QRecord().withValue("createDate", Instant.parse("2024-04-04T19:12:00Z"));
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, qInstance, table, List.of(record), null);
+      assertEquals("2024-04-04 12:12:00 PM PDT", record.getDisplayValue("createDate"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testValidation()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      QFieldMetaData field = table.getField("createDate");
+      table.withField(new QFieldMetaData("timeZone", QFieldType.STRING));
+
+      Function<Consumer<DateTimeDisplayValueBehavior>, List<String>> testOne = setup ->
+      {
+         DateTimeDisplayValueBehavior dateTimeDisplayValueBehavior = new DateTimeDisplayValueBehavior();
+         setup.accept(dateTimeDisplayValueBehavior);
+         return (dateTimeDisplayValueBehavior.validateBehaviorConfiguration(table, field));
+      };
+
+      ///////////////////
+      // valid configs //
+      ///////////////////
+      assertThat(testOne.apply(b -> b.toString())).isEmpty(); // default setup (noop use-case) is valid
+      assertThat(testOne.apply(b -> b.withZoneIdFromFieldName("timeZone"))).isEmpty();
+      assertThat(testOne.apply(b -> b.withZoneIdFromFieldName("timeZone").withFallbackZoneId("UTC"))).isEmpty();
+      assertThat(testOne.apply(b -> b.withZoneIdFromFieldName("timeZone").withFallbackZoneId("America/Chicago"))).isEmpty();
+      assertThat(testOne.apply(b -> b.withDefaultZoneId("UTC"))).isEmpty();
+      assertThat(testOne.apply(b -> b.withDefaultZoneId("America/Chicago"))).isEmpty();
+
+      /////////////////////
+      // invalid configs //
+      /////////////////////
+      assertThat(testOne.apply(b -> b.withZoneIdFromFieldName("notAField")))
+         .hasSize(1).first().asString()
+         .contains("Unrecognized field name");
+
+      assertThat(testOne.apply(b -> b.withZoneIdFromFieldName("id")))
+         .hasSize(1).first().asString()
+         .contains("A non-STRING type [INTEGER] was specified as the zoneIdFromFieldName field");
+
+      assertThat(testOne.apply(b -> b.withZoneIdFromFieldName("timeZone").withDefaultZoneId("UTC")))
+         .hasSize(1).first().asString()
+         .contains("You may not specify both zoneIdFromFieldName and defaultZoneId");
+
+      assertThat(testOne.apply(b -> b.withDefaultZoneId("UTC").withFallbackZoneId("UTC")))
+         .hasSize(2)
+         .anyMatch(s -> s.contains("You may not specify both defaultZoneId and fallbackZoneId"))
+         .anyMatch(s -> s.contains("You may only set fallbackZoneId if using zoneIdFromFieldName"));
+
+      assertThat(testOne.apply(b -> b.withFallbackZoneId("UTC")))
+         .hasSize(1).first().asString()
+         .contains("You may only set fallbackZoneId if using zoneIdFromFieldName");
+
+      assertThat(testOne.apply(b -> b.withDefaultZoneId("notAZone")))
+         .hasSize(1).first().asString()
+         .contains("Invalid ZoneId [notAZone] for [defaultZoneId]");
+
+      assertThat(testOne.apply(b -> b.withZoneIdFromFieldName("timeZone").withFallbackZoneId("notAZone")))
+         .hasSize(1).first().asString()
+         .contains("Invalid ZoneId [notAZone] for [fallbackZoneId]");
+
+      assertThat(new DateTimeDisplayValueBehavior().validateBehaviorConfiguration(table, table.getField("firstName")))
+         .hasSize(1).first().asString()
+         .contains("non-DATE_TIME field [firstName]");
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java

@@ -22,6 +22,8 @@
 package com.kingsrook.qqq.backend.core.state;
 
 
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.UUID;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -88,4 +90,42 @@ public class InMemoryStateProviderTest extends BaseTest
       });
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testClean()
+   {
+      InMemoryStateProvider stateProvider = InMemoryStateProvider.getInstance();
+
+      /////////////////////////////////////////////////////////////
+      // Add an entry that is 3 hours old, should not be cleaned //
+      /////////////////////////////////////////////////////////////
+      UUIDAndTypeStateKey   newKey           = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(3, ChronoUnit.HOURS));
+      String  newUUID    = UUID.randomUUID().toString();
+      QRecord newQRecord = new QRecord().withValue("uuid", newUUID);
+      stateProvider.put(newKey, newQRecord);
+
+      ////////////////////////////////////////////////////////////
+      // Add an entry that is 5 hours old, it should be cleaned //
+      ////////////////////////////////////////////////////////////
+      UUIDAndTypeStateKey   oldKey           = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(5, ChronoUnit.HOURS));
+      String  oldUUID    = UUID.randomUUID().toString();
+      QRecord oldQRecord = new QRecord().withValue("uuid", oldUUID);
+      stateProvider.put(oldKey, oldQRecord);
+
+      ///////////////////
+      // Call to clean //
+      ///////////////////
+      stateProvider.clean(Instant.now().minus(4, ChronoUnit.HOURS));
+
+      QRecord qRecordFromState = stateProvider.get(QRecord.class, newKey).get();
+      Assertions.assertEquals(newUUID, qRecordFromState.getValueString("uuid"), "Should read value from state persistence");
+
+      Assertions.assertTrue(stateProvider.get(QRecord.class, oldKey).isEmpty(), "Key not found in state should return empty");
+
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -138,6 +138,7 @@ public class TestUtils
    public static final String APP_NAME_PEOPLE        = "peopleApp";
    public static final String APP_NAME_MISCELLANEOUS = "miscellaneous";
 
+   public static final String TABLE_NAME_TWO_KEYS            = "twoKeys";
    public static final String TABLE_NAME_PERSON              = "person";
    public static final String TABLE_NAME_SHAPE               = "shape";
    public static final String TABLE_NAME_SHAPE_CACHE         = "shapeCache";
@@ -196,6 +197,7 @@ public class TestUtils
       qInstance.addBackend(defineMemoryBackend());
 
       qInstance.addTable(defineTablePerson());
+      qInstance.addTable(defineTableTwoKeys());
       qInstance.addTable(definePersonFileTable());
       qInstance.addTable(definePersonMemoryTable());
       qInstance.addTable(definePersonMemoryCacheTable());
@@ -545,6 +547,24 @@ public class TestUtils
 
 
 
+   /*******************************************************************************
+    ** Define the 'two key' table used in standard tests.
+    *******************************************************************************/
+   public static QTableMetaData defineTableTwoKeys()
+   {
+      return new QTableMetaData()
+         .withName(TABLE_NAME_TWO_KEYS)
+         .withLabel("Two Keys")
+         .withBackendName(MEMORY_BACKEND_NAME)
+         .withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withUniqueKey(new UniqueKey("key1", "key2"))
+         .withField(new QFieldMetaData("key1", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("key2", QFieldType.INTEGER));
+   }
+
+
+
    /*******************************************************************************
     ** Define the 'person' table used in standard tests.
     *******************************************************************************/
@@ -791,6 +811,26 @@ public class TestUtils
 
 
 
+   /*******************************************************************************
+    ** Define a table with unique key where one is nullable
+    *******************************************************************************/
+   public static QTableMetaData defineTwoKeyTable()
+   {
+      return (new QTableMetaData()
+         .withName(TABLE_NAME_BASEPULL)
+         .withLabel("Basepull Test")
+         .withPrimaryKeyField("id")
+         .withBackendName(MEMORY_BACKEND_NAME)
+         .withFields(TestUtils.defineTablePerson().getFields()))
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date").withIsEditable(false))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date").withIsEditable(false))
+         .withField(new QFieldMetaData(BASEPULL_KEY_FIELD_NAME, QFieldType.STRING).withBackendName("process_name").withIsRequired(true))
+         .withField(new QFieldMetaData(BASEPULL_LAST_RUN_TIME_FIELD_NAME, QFieldType.DATE_TIME).withBackendName("last_run_time").withIsRequired(true));
+   }
+
+
+
    /*******************************************************************************
     ** Define a basepullTable
     *******************************************************************************/

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java

@@ -49,6 +49,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractF
 import com.kingsrook.qqq.backend.module.filesystem.base.utils.SharedFilesystemBackendModuleUtils;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import org.apache.commons.io.FileUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -183,7 +184,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
          // if the file doesn't exist, just exit with noop.  don't throw an error - that should only //
          // happen if the "contract" of the method is broken, and the file still exists              //
          //////////////////////////////////////////////////////////////////////////////////////////////
-         LOG.debug("Not deleting file [{}], because it does not exist.", file);
+         LOG.debug("Not deleting file, because it does not exist.", logPair("file", file));
          return;
       }
 
@@ -218,7 +219,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
       //////////////////////////////////////////////////////////////////////////////////////
       if(!destinationParent.exists())
       {
-         LOG.debug("Making destination directory {} for move", destinationParent.getAbsolutePath());
+         LOG.debug("Making destination directory for move", logPair("directory", destinationParent.getAbsolutePath()));
          if(!destinationParent.mkdirs())
          {
             throw (new FilesystemException("Failed to make destination directory " + destinationParent.getAbsolutePath() + " to move " + source + " into."));

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java

@@ -357,16 +357,22 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
     *******************************************************************************/
    private PreparedStatement createStatement(Connection connection, String sql, QueryInput queryInput) throws SQLException
    {
-      if(mysqlResultSetOptimizationEnabled && connection.getClass().getName().startsWith("com.mysql"))
+      if(connection.getClass().getName().startsWith("com.mysql"))
       {
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html //
+         // if we're allowed to use the mysqlResultSetOptimization, and we have the query hint of "expected large result set", then do it. //
-         // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows).                 //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // with this change, we start to get results immediately, and the total runtime also seems lower...                                                            //
+         if(mysqlResultSetOptimizationEnabled && queryInput.getQueryHints() != null && queryInput.getQueryHints().contains(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS))
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         {
-         PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         statement.setFetchSize(Integer.MIN_VALUE);
+            // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html //
-         return (statement);
+            // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows).                 //
+            // with this change, we start to get results immediately, and the total runtime also seems lower...                                                            //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+            statement.setFetchSize(Integer.MIN_VALUE);
+            return (statement);
+         }
       }
 
       return (connection.prepareStatement(sql));

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java

@@ -1403,7 +1403,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       {
          String           associatedTableName        = association.getAssociatedTableName();
          QTableMetaData   associatedTable            = QContext.getQInstance().getTable(associatedTableName);
-         ApiTableMetaData associatedApiTableMetaData = ObjectUtils.tryElse(() -> ApiTableMetaDataContainer.of(associatedTable).getApiTableMetaData(apiName), new ApiTableMetaData());
+         ApiTableMetaData associatedApiTableMetaData = ObjectUtils.tryAndRequireNonNullElse(() -> ApiTableMetaDataContainer.of(associatedTable).getApiTableMetaData(apiName), new ApiTableMetaData());
          String           associatedTableApiName     = StringUtils.hasContent(associatedApiTableMetaData.getApiTableName()) ? associatedApiTableMetaData.getApiTableName() : associatedTableName;
 
          ApiAssociationMetaData apiAssociationMetaData = thisApiTableMetaData.getApiAssociationMetaData().get(association.getName());

qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java

@@ -662,11 +662,11 @@ public class QSlackImplementation
          //////////////////////////////////////////////////////////////////////////
          // Print result, which includes information about the message (like TS) //
          //////////////////////////////////////////////////////////////////////////
-         LOG.info("Slack post result {}", result);
+         LOG.info("Slack post result: " + result);
       }
       catch(IOException | SlackApiException e)
       {
-         LOG.error("error: {}", e.getMessage(), e);
+         LOG.error("error", e);
       }
    }