updates to check versions on process query params

Feature/string utils safe equals ignore case

pom.xml

@@ -48,7 +48,7 @@
    </modules>
 
    <properties>
-      <revision>0.24.0</revision>
+      <revision>0.25.0-SNAPSHOT</revision>
 
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

qqq-backend-core/pom.xml

@@ -65,7 +65,11 @@
          <artifactId>aws-java-sdk-secretsmanager</artifactId>
          <version>1.12.385</version>
       </dependency>
-
+      <dependency>
+         <groupId>com.ibm.icu</groupId>
+         <artifactId>icu4j</artifactId>
+         <version>77.1</version>
+      </dependency>
       <dependency>
          <groupId>com.fasterxml.jackson.core</groupId>
          <artifactId>jackson-databind</artifactId>
@@ -151,16 +155,21 @@
          <version>2.3</version>
       </dependency>
 
-      <!-- the next 2 deps are for html to pdf - per https://www.baeldung.com/java-html-to-pdf -->
+      <!-- the next 3 deps are for html to pdf -->
       <dependency>
          <groupId>org.jsoup</groupId>
          <artifactId>jsoup</artifactId>
          <version>1.15.3</version>
       </dependency>
       <dependency>
-         <groupId>org.xhtmlrenderer</groupId>
-         <artifactId>flying-saucer-pdf-openpdf</artifactId>
-         <version>9.1.22</version>
+         <groupId>com.openhtmltopdf</groupId>
+         <artifactId>openhtmltopdf-core</artifactId>
+         <version>1.0.10</version>
+      </dependency>
+      <dependency>
+         <groupId>com.openhtmltopdf</groupId>
+         <artifactId>openhtmltopdf-pdfbox</artifactId>
+         <version>1.0.10</version>
       </dependency>
 
       <!-- the next 3 deps are being added for google drive support -->

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RunRecordScriptAutomationHandler.java

@@ -83,7 +83,7 @@ public class RunRecordScriptAutomationHandler extends RecordAutomationHandler
       }
 
       QRecord scriptRevision = queryOutput.getRecords().get(0);
-      LOG.info("Running script against records", logPair("scriptRevisionId", scriptRevision.getValue("id")), logPair("scriptId", scriptRevision.getValue("scriptIdd")));
+      LOG.debug("Running script against records", logPair("scriptRevisionId", scriptRevision.getValue("id")), logPair("scriptId", scriptRevision.getValue("scriptIdd")));
 
       RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();
       input.setCodeReference(new AdHocScriptCodeReference().withScriptRevisionRecord(scriptRevision));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -649,7 +649,7 @@ public class PollingAutomationPerTableRunner implements Runnable
          input.setRecordList(records);
          input.setAction(action);
 
-         RecordAutomationHandler recordAutomationHandler = QCodeLoader.getRecordAutomationHandler(action);
+         RecordAutomationHandler recordAutomationHandler = QCodeLoader.getAdHoc(RecordAutomationHandler.class, action.getCodeReference());
          recordAutomationHandler.execute(input);
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java

@@ -24,17 +24,11 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
 
 import java.lang.reflect.Constructor;
 import java.util.Optional;
-import java.util.function.Function;
-import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
-import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
-import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.code.InitializableViaCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
-import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
 import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
 import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -43,9 +37,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 /*******************************************************************************
  ** Utility to load code for running QQQ customizers.
  **
- ** TODO - redo all to go through method that memoizes class & constructor
- ** lookup.  That memoziation causes 1,000,000 such calls to go from ~500ms
- ** to ~100ms.
+ ** That memoization causes 1,000,000 such calls to go from ~500ms to ~100ms.
  *******************************************************************************/
 public class QCodeLoader
 {
@@ -70,84 +62,6 @@ public class QCodeLoader
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @SuppressWarnings("unchecked")
-   public static <T, R> Function<T, R> getFunction(QCodeReference codeReference)
-   {
-      if(codeReference == null)
-      {
-         return (null);
-      }
-
-      if(!codeReference.getCodeType().equals(QCodeType.JAVA))
-      {
-         ///////////////////////////////////////////////////////////////////////////////////////
-         // todo - 1) support more languages, 2) wrap them w/ java Functions here, 3) profit! //
-         ///////////////////////////////////////////////////////////////////////////////////////
-         throw (new IllegalArgumentException("Only JAVA customizers are supported at this time."));
-      }
-
-      try
-      {
-         Class<?> customizerClass = Class.forName(codeReference.getName());
-         return ((Function<T, R>) customizerClass.getConstructor().newInstance());
-      }
-      catch(Exception e)
-      {
-         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
-
-         //////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // return null here - under the assumption that during normal run-time operations, we'll never hit here //
-         // as we'll want to validate all functions in the instance validator at startup time (and IT will throw //
-         // if it finds an invalid code reference                                                                //
-         //////////////////////////////////////////////////////////////////////////////////////////////////////////
-         return (null);
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @SuppressWarnings("unchecked")
-   public static <T extends BackendStep> T getBackendStep(Class<T> expectedType, QCodeReference codeReference)
-   {
-      if(codeReference == null)
-      {
-         return (null);
-      }
-
-      if(!codeReference.getCodeType().equals(QCodeType.JAVA))
-      {
-         ///////////////////////////////////////////////////////////////////////////////////////
-         // todo - 1) support more languages, 2) wrap them w/ java Functions here, 3) profit! //
-         ///////////////////////////////////////////////////////////////////////////////////////
-         throw (new IllegalArgumentException("Only JAVA BackendSteps are supported at this time."));
-      }
-
-      try
-      {
-         Class<?> customizerClass = Class.forName(codeReference.getName());
-         return ((T) customizerClass.getConstructor().newInstance());
-      }
-      catch(Exception e)
-      {
-         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
-
-         //////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // return null here - under the assumption that during normal run-time operations, we'll never hit here //
-         // as we'll want to validate all functions in the instance validator at startup time (and IT will throw //
-         // if it finds an invalid code reference                                                                //
-         //////////////////////////////////////////////////////////////////////////////////////////////////////////
-         return (null);
-      }
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -177,7 +91,17 @@ public class QCodeLoader
 
          if(constructor.isPresent())
          {
-            return ((T) constructor.get().newInstance());
+            T t = (T) constructor.get().newInstance();
+
+            ////////////////////////////////////////////////////////////////
+            // if the object is initializable, then, well, initialize it! //
+            ////////////////////////////////////////////////////////////////
+            if(t instanceof InitializableViaCodeReference initializableViaCodeReference)
+            {
+               initializableViaCodeReference.initialize(codeReference);
+            }
+
+            return t;
          }
          else
          {
@@ -187,7 +111,7 @@ public class QCodeLoader
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
+         LOG.error("Error initializing codeReference", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //
@@ -198,67 +122,4 @@ public class QCodeLoader
       }
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static RecordAutomationHandler getRecordAutomationHandler(TableAutomationAction action) throws QException
-   {
-      try
-      {
-         QCodeReference codeReference = action.getCodeReference();
-         if(!codeReference.getCodeType().equals(QCodeType.JAVA))
-         {
-            ///////////////////////////////////////////////////////////////////////////////////////
-            // todo - 1) support more languages, 2) wrap them w/ java Functions here, 3) profit! //
-            ///////////////////////////////////////////////////////////////////////////////////////
-            throw (new IllegalArgumentException("Only JAVA customizers are supported at this time."));
-         }
-
-         Class<?> codeClass  = Class.forName(codeReference.getName());
-         Object   codeObject = codeClass.getConstructor().newInstance();
-         if(!(codeObject instanceof RecordAutomationHandler recordAutomationHandler))
-         {
-            throw (new QException("The supplied code [" + codeClass.getName() + "] is not an instance of RecordAutomationHandler"));
-         }
-         return (recordAutomationHandler);
-      }
-      catch(QException qe)
-      {
-         throw (qe);
-      }
-      catch(Exception e)
-      {
-         throw (new QException("Error getting record automation handler for action [" + action.getName() + "]", e));
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static QCustomPossibleValueProvider getCustomPossibleValueProvider(QPossibleValueSource possibleValueSource) throws QException
-   {
-      try
-      {
-         Class<?> codeClass  = Class.forName(possibleValueSource.getCustomCodeReference().getName());
-         Object   codeObject = codeClass.getConstructor().newInstance();
-         if(!(codeObject instanceof QCustomPossibleValueProvider customPossibleValueProvider))
-         {
-            throw (new QException("The supplied code [" + codeClass.getName() + "] is not an instance of QCustomPossibleValueProvider"));
-         }
-         return (customPossibleValueProvider);
-      }
-      catch(QException qe)
-      {
-         throw (qe);
-      }
-      catch(Exception e)
-      {
-         throw (new QException("Error getting custom possible value provider for PVS [" + possibleValueSource.getName() + "]", e));
-      }
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractHTMLWidgetRenderer.java

@@ -290,7 +290,18 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
+   @Deprecated(since = "call one that doesn't take input param")
    public static String linkRecordEdit(AbstractActionInput input, String tableName, Serializable recordId) throws QException
+   {
+      return linkRecordEdit(tableName, recordId);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String linkRecordEdit(String tableName, Serializable recordId) throws QException
    {
       String tablePath = QContext.getQInstance().getTablePath(tableName);
       return (tablePath + "/" + recordId + "/edit");
@@ -317,7 +328,17 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
+   @Deprecated(since = "call one that doesn't take input param")
    public static String linkProcessForFilter(AbstractActionInput input, String processName, QQueryFilter filter) throws QException
+   {
+      return linkProcessForFilter(processName, filter);
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String linkProcessForFilter(String processName, QQueryFilter filter) throws QException
    {
       QProcessMetaData process = QContext.getQInstance().getProcess(processName);
       if(process == null)
@@ -337,10 +358,21 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
 
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
+   @Deprecated(since = "call one that doesn't take input param")
    public static String linkProcessForRecord(AbstractActionInput input, String processName, Serializable recordId) throws QException
+   {
+      return linkProcessForRecord(processName, recordId);
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String linkProcessForRecord(String processName, Serializable recordId) throws QException
    {
       QProcessMetaData process   = QContext.getQInstance().getProcess(processName);
       String           tableName = process.getTableName();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/AllowAllMetaDataFilter.java

@@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 /*******************************************************************************
  ** a default implementation of MetaDataFilterInterface, that allows all the things
  *******************************************************************************/
+@Deprecated(since = "migrated to metaDataCustomizer")
 public class AllowAllMetaDataFilter implements MetaDataFilterInterface
 {
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/DefaultNoopMetaDataActionCustomizer.java

@@ -0,0 +1,92 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.metadata;
+
+
+import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** a default implementation of MetaDataFilterInterface, that is all noop.
+ *******************************************************************************/
+public class DefaultNoopMetaDataActionCustomizer implements MetaDataActionCustomizerInterface
+{
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public boolean allowTable(MetaDataInput input, QTableMetaData table)
+   {
+      return (true);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public boolean allowProcess(MetaDataInput input, QProcessMetaData process)
+   {
+      return (true);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public boolean allowReport(MetaDataInput input, QReportMetaData report)
+   {
+      return (true);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public boolean allowApp(MetaDataInput input, QAppMetaData app)
+   {
+      return (true);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public boolean allowWidget(MetaDataInput input, QWidgetMetaDataInterface widget)
+   {
+      return (true);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java

@@ -23,10 +23,12 @@ package com.kingsrook.qqq.backend.core.actions.metadata;
 
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult;
@@ -34,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput;
@@ -65,7 +68,7 @@ public class MetaDataAction
 {
    private static final QLogger LOG = QLogger.getLogger(MetaDataAction.class);
 
-   private static Memoization<QInstance, MetaDataFilterInterface> metaDataFilterMemoization = new Memoization<>();
+   private static Memoization<QInstance, MetaDataActionCustomizerInterface> metaDataActionCustomizerMemoization = new Memoization<>();
 
 
 
@@ -79,7 +82,7 @@ public class MetaDataAction
       MetaDataOutput           metaDataOutput = new MetaDataOutput();
       Map<String, AppTreeNode> treeNodes      = new LinkedHashMap<>();
 
-      MetaDataFilterInterface filter = getMetaDataFilter();
+      MetaDataActionCustomizerInterface customizer = getMetaDataActionCustomizer();
 
       /////////////////////////////////////
       // map tables to frontend metadata //
@@ -90,7 +93,7 @@ public class MetaDataAction
          String         tableName = entry.getKey();
          QTableMetaData table     = entry.getValue();
 
-         if(!filter.allowTable(metaDataInput, table))
+         if(!customizer.allowTable(metaDataInput, table))
          {
             continue;
          }
@@ -119,7 +122,7 @@ public class MetaDataAction
          String           processName = entry.getKey();
          QProcessMetaData process     = entry.getValue();
 
-         if(!filter.allowProcess(metaDataInput, process))
+         if(!customizer.allowProcess(metaDataInput, process))
          {
             continue;
          }
@@ -144,7 +147,7 @@ public class MetaDataAction
          String          reportName = entry.getKey();
          QReportMetaData report     = entry.getValue();
 
-         if(!filter.allowReport(metaDataInput, report))
+         if(!customizer.allowReport(metaDataInput, report))
          {
             continue;
          }
@@ -169,7 +172,7 @@ public class MetaDataAction
          String                   widgetName = entry.getKey();
          QWidgetMetaDataInterface widget     = entry.getValue();
 
-         if(!filter.allowWidget(metaDataInput, widget))
+         if(!customizer.allowWidget(metaDataInput, widget))
          {
             continue;
          }
@@ -206,7 +209,7 @@ public class MetaDataAction
             continue;
          }
 
-         if(!filter.allowApp(metaDataInput, app))
+         if(!customizer.allowApp(metaDataInput, app))
          {
             continue;
          }
@@ -292,11 +295,22 @@ public class MetaDataAction
          metaDataOutput.setBranding(QContext.getQInstance().getBranding());
       }
 
-      metaDataOutput.setEnvironmentValues(QContext.getQInstance().getEnvironmentValues());
+      metaDataOutput.setEnvironmentValues(Objects.requireNonNullElse(QContext.getQInstance().getEnvironmentValues(), Collections.emptyMap()));
 
-      metaDataOutput.setHelpContents(QContext.getQInstance().getHelpContent());
+      metaDataOutput.setHelpContents(Objects.requireNonNullElse(QContext.getQInstance().getHelpContent(), Collections.emptyMap()));
 
-      // todo post-customization - can do whatever w/ the result if you want?
+      try
+      {
+         customizer.postProcess(metaDataOutput);
+      }
+      catch(QUserFacingException e)
+      {
+         LOG.debug("User-facing exception thrown in meta-data customizer post-processing", e);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Unexpected error thrown in meta-data customizer post-processing", e);
+      }
 
       return metaDataOutput;
    }
@@ -306,26 +320,36 @@ public class MetaDataAction
    /***************************************************************************
     **
     ***************************************************************************/
-   private MetaDataFilterInterface getMetaDataFilter()
+   private MetaDataActionCustomizerInterface getMetaDataActionCustomizer()
    {
-      return metaDataFilterMemoization.getResult(QContext.getQInstance(), i ->
+      return metaDataActionCustomizerMemoization.getResult(QContext.getQInstance(), i ->
       {
-         MetaDataFilterInterface filter                  = null;
-         QCodeReference          metaDataFilterReference = QContext.getQInstance().getMetaDataFilter();
-         if(metaDataFilterReference != null)
+         MetaDataActionCustomizerInterface actionCustomizer                  = null;
+         QCodeReference                    metaDataActionCustomizerReference = QContext.getQInstance().getMetaDataActionCustomizer();
+         if(metaDataActionCustomizerReference != null)
          {
-            filter = QCodeLoader.getAdHoc(MetaDataFilterInterface.class, metaDataFilterReference);
-            LOG.debug("Using new meta-data filter of type: " + filter.getClass().getSimpleName());
+            actionCustomizer = QCodeLoader.getAdHoc(MetaDataActionCustomizerInterface.class, metaDataActionCustomizerReference);
+            LOG.debug("Using new meta-data actionCustomizer of type: " + actionCustomizer.getClass().getSimpleName());
          }
 
-         if(filter == null)
+         if(actionCustomizer == null)
          {
-            filter = new AllowAllMetaDataFilter();
-            LOG.debug("Using new default (allow-all) meta-data filter");
+            QCodeReference metaDataFilterReference = QContext.getQInstance().getMetaDataFilter();
+            if(metaDataFilterReference != null)
+            {
+               actionCustomizer = QCodeLoader.getAdHoc(MetaDataActionCustomizerInterface.class, metaDataFilterReference);
+               LOG.debug("Using new meta-data actionCustomizer (via metaDataFilter reference) of type: " + actionCustomizer.getClass().getSimpleName());
+            }
          }
 
-         return (filter);
-      }).orElseThrow(() -> new QRuntimeException("Error getting metaDataFilter"));
+         if(actionCustomizer == null)
+         {
+            actionCustomizer = new DefaultNoopMetaDataActionCustomizer();
+            LOG.debug("Using new default (allow-all) meta-data actionCustomizer");
+         }
+
+         return (actionCustomizer);
+      }).orElseThrow(() -> new QRuntimeException("Error getting MetaDataActionCustomizer"));
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionCustomizerInterface.java

@@ -0,0 +1,78 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput;
+import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** Interface for customizations that can be injected by an application into
+ ** the MetaDataAction - e.g., loading applicable meta-data for a user into a
+ ** frontend.
+ *******************************************************************************/
+public interface MetaDataActionCustomizerInterface
+{
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   boolean allowTable(MetaDataInput input, QTableMetaData table);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   boolean allowProcess(MetaDataInput input, QProcessMetaData process);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   boolean allowReport(MetaDataInput input, QReportMetaData report);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   boolean allowApp(MetaDataInput input, QAppMetaData app);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   boolean allowWidget(MetaDataInput input, QWidgetMetaDataInterface widget);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   default void postProcess(MetaDataOutput metaDataOutput) throws QException
+   {
+      /////////////////////
+      // noop by default //
+      /////////////////////
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataFilterInterface.java

@@ -22,43 +22,11 @@
 package com.kingsrook.qqq.backend.core.actions.metadata;
 
 
-import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput;
-import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
-import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-
-
 /*******************************************************************************
  **
  *******************************************************************************/
-public interface MetaDataFilterInterface
+@Deprecated(since = "migrated to metaDataCustomizer")
+public interface MetaDataFilterInterface extends MetaDataActionCustomizerInterface
 {
 
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   boolean allowTable(MetaDataInput input, QTableMetaData table);
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   boolean allowProcess(MetaDataInput input, QProcessMetaData process);
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   boolean allowReport(MetaDataInput input, QReportMetaData report);
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   boolean allowApp(MetaDataInput input, QAppMetaData app);
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   boolean allowWidget(MetaDataInput input, QWidgetMetaDataInterface widget);
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java

@@ -36,6 +36,8 @@ import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
@@ -46,6 +48,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
@@ -188,21 +191,40 @@ public class RunBackendStepAction
       {
          if(CollectionUtils.nullSafeIsEmpty(runBackendStepInput.getRecords()))
          {
-            QueryInput queryInput = new QueryInput();
-            queryInput.setTableName(inputMetaData.getRecordListMetaData().getTableName());
+            QTableMetaData table      = QContext.getQInstance().getTable(inputMetaData.getRecordListMetaData().getTableName());
+            QueryInput     queryInput = new QueryInput();
+            queryInput.setTableName(table.getName());
 
-            // todo - handle this being async (e.g., http)
-            // seems like it just needs to throw, breaking this flow, and to send a response to the frontend, directing it to prompt the user for the needed data
-            // then this step can re-run, hopefully with the needed data.
-
-            QProcessCallback callback = runBackendStepInput.getCallback();
-            if(callback == null)
+            //////////////////////////////////////////////////
+            // look for record ids in the input data values //
+            //////////////////////////////////////////////////
+            String recordIds = (String) runBackendStepInput.getValue("recordIds");
+            if(recordIds == null)
             {
-               throw (new QUserFacingException("Missing input records.",
-                  new QException("Function is missing input records, but no callback was present to request fields from a user")));
+               recordIds = (String) runBackendStepInput.getValue("recordId");
             }
 
-            queryInput.setFilter(callback.getQueryFilter());
+            ///////////////////////////////////////////////////////////
+            // if records were found, add as criteria to query input //
+            ///////////////////////////////////////////////////////////
+            if(recordIds != null)
+            {
+               queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordIds.split(","))));
+            }
+            else
+            {
+               // todo - handle this being async (e.g., http)
+               // seems like it just needs to throw, breaking this flow, and to send a response to the frontend, directing it to prompt the user for the needed data
+               // then this step can re-run, hopefully with the needed data.
+               QProcessCallback callback = runBackendStepInput.getCallback();
+               if(callback == null)
+               {
+                  throw (new QUserFacingException("Missing input records.",
+                     new QException("Function is missing input records, but no callback was present to request fields from a user")));
+               }
+
+               queryInput.setFilter(callback.getQueryFilter());
+            }
 
             //////////////////////////////////////////////////////////////////////////////////////////
             // if process has a max-no of records, set a limit on the process of that number plus 1 //
@@ -210,7 +232,7 @@ public class RunBackendStepAction
             //////////////////////////////////////////////////////////////////////////////////////////
             if(process.getMaxInputRecords() != null)
             {
-               if(callback.getQueryFilter() == null)
+               if(queryInput.getFilter() == null)
                {
                   queryInput.setFilter(new QQueryFilter());
                }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportStreamerInterface.java

@@ -54,6 +54,14 @@ public interface ExportStreamerInterface
       // noop in base class
    }
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   default void setExportStyleCustomizer(ExportStyleCustomizerInterface exportStyleCustomizer)
+   {
+      // noop in base class
+   }
+
    /*******************************************************************************
     ** Called once per sheet, before any rows are available.  Meant to write a
     ** header, for example.

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportStyleCustomizerInterface.java

@@ -0,0 +1,35 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+/*******************************************************************************
+ ** interface for classes that can be used to customize visual style aspects of
+ ** exports/reports.
+ **
+ ** Anticipates very different sub-interfaces based on the file type being generated,
+ ** and the capabilities of each.  e.g., excel (bolds, fonts, cell merging) vs
+ ** json (different structure of objects).
+ *******************************************************************************/
+public interface ExportStyleCustomizerInterface
+{
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -163,6 +163,17 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
          reportStreamer = reportFormat.newReportStreamer();
       }
 
+      if(reportInput.getExportStyleCustomizer() != null)
+      {
+         ExportStyleCustomizerInterface styleCustomizer = QCodeLoader.getAdHoc(ExportStyleCustomizerInterface.class, reportInput.getExportStyleCustomizer());
+         reportStreamer.setExportStyleCustomizer(styleCustomizer);
+      }
+      else if(report.getExportStyleCustomizer() != null)
+      {
+         ExportStyleCustomizerInterface styleCustomizer = QCodeLoader.getAdHoc(ExportStyleCustomizerInterface.class, report.getExportStyleCustomizer());
+         reportStreamer.setExportStyleCustomizer(styleCustomizer);
+      }
+
       reportStreamer.preRun(reportInput.getReportDestination(), views);
 
       ////////////////////////////////////////////////////////////////////////////////////////////////
@@ -211,7 +222,8 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
                /////////////////////////////////////////////////////////////////////////////////////////
                if(dataSourceTableView.getViewCustomizer() != null)
                {
-                  Function<QReportView, QReportView> viewCustomizerFunction = QCodeLoader.getFunction(dataSourceTableView.getViewCustomizer());
+                  @SuppressWarnings("unchecked")
+                  Function<QReportView, QReportView> viewCustomizerFunction = QCodeLoader.getAdHoc(Function.class, dataSourceTableView.getViewCustomizer());
                   if(viewCustomizerFunction instanceof ReportViewCustomizer reportViewCustomizer)
                   {
                      reportViewCustomizer.setReportInput(reportInput);
@@ -660,7 +672,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
          ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
          // if any fields are 'showPossibleValueLabel', then move display values for them into the record's values map //
          ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         for(QReportField column : tableView.getColumns())
+         for(QReportField column : CollectionUtils.nonNullList(tableView.getColumns()))
          {
             if(column.getShowPossibleValueLabel())
             {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java

@@ -46,6 +46,7 @@ import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 import java.util.zip.ZipOutputStream;
 import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface;
+import com.kingsrook.qqq.backend.core.actions.reporting.ExportStyleCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.reporting.ReportUtils;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
@@ -77,6 +78,7 @@ import org.apache.poi.xssf.usermodel.XSSFPivotTable;
 import org.apache.poi.xssf.usermodel.XSSFRow;
 import org.apache.poi.xssf.usermodel.XSSFSheet;
 import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -112,9 +114,10 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
    public static final String EXCEL_DATE_FORMAT      = "yyyy-MM-dd";
    public static final String EXCEL_DATE_TIME_FORMAT = "yyyy-MM-dd H:mm:ss";
 
-   private PoiExcelStylerInterface    poiExcelStylerInterface = getStylerInterface();
+   private ExcelPoiBasedStreamingStyleCustomizerInterface styleCustomizerInterface;
+
    private Map<String, String>        excelCellFormats;
-   private Map<String, XSSFCellStyle> styles                  = new HashMap<>();
+   private Map<String, XSSFCellStyle> styles = new HashMap<>();
 
    private int rowNo      = 0;
    private int sheetIndex = 1;
@@ -402,6 +405,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
       dateTimeStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_TIME_FORMAT));
       styles.put("datetime", dateTimeStyle);
 
+      PoiExcelStylerInterface poiExcelStylerInterface = getStylerInterface();
       styles.put("title", poiExcelStylerInterface.createStyleForTitle(workbook, createHelper));
       styles.put("header", poiExcelStylerInterface.createStyleForHeader(workbook, createHelper));
       styles.put("footer", poiExcelStylerInterface.createStyleForFooter(workbook, createHelper));
@@ -413,6 +417,11 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
       XSSFCellStyle footerDateTimeStyle = poiExcelStylerInterface.createStyleForFooter(workbook, createHelper);
       footerDateTimeStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_TIME_FORMAT));
       styles.put("footer-datetime", footerDateTimeStyle);
+
+      if(styleCustomizerInterface != null)
+      {
+         styleCustomizerInterface.customizeStyles(styles, workbook, createHelper);
+      }
    }
 
 
@@ -458,7 +467,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
          }
          else
          {
-            sheetWriter.beginSheet();
+            sheetWriter.beginSheet(view, styleCustomizerInterface);
 
             ////////////////////////////////////////////////
             // put the title and header rows in the sheet //
@@ -560,6 +569,16 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static void setStyleForField(QRecord record, String fieldName, String styleName)
+   {
+      record.setDisplayValue(fieldName + ":excelStyle", styleName);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -567,12 +586,12 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
    {
       sheetWriter.insertRow(rowNo++);
 
-      int styleIndex         = -1;
+      int baseStyleIndex     = -1;
       int dateStyleIndex     = styles.get("date").getIndex();
       int dateTimeStyleIndex = styles.get("datetime").getIndex();
       if(isFooter)
       {
-         styleIndex = styles.get("footer").getIndex();
+         baseStyleIndex = styles.get("footer").getIndex();
          dateStyleIndex = styles.get("footer-date").getIndex();
          dateTimeStyleIndex = styles.get("footer-datetime").getIndex();
       }
@@ -582,6 +601,13 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
       {
          Serializable value = qRecord.getValue(field.getName());
 
+         String overrideStyleName = qRecord.getDisplayValue(field.getName() + ":excelStyle");
+         int    styleIndex        = baseStyleIndex;
+         if(overrideStyleName != null)
+         {
+            styleIndex = styles.get(overrideStyleName).getIndex();
+         }
+
          if(value != null)
          {
             if(value instanceof String s)
@@ -706,7 +732,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
       {
          if(!ReportType.PIVOT.equals(currentView.getType()))
          {
-            sheetWriter.endSheet();
+            sheetWriter.endSheet(currentView, styleCustomizerInterface);
          }
 
          activeSheetWriter.flush();
@@ -815,7 +841,29 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
     *******************************************************************************/
    protected PoiExcelStylerInterface getStylerInterface()
    {
+      if(styleCustomizerInterface != null)
+      {
+         return styleCustomizerInterface.getExcelStyler();
+      }
+
       return (new PlainPoiExcelStyler());
    }
 
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void setExportStyleCustomizer(ExportStyleCustomizerInterface exportStyleCustomizer)
+   {
+      if(exportStyleCustomizer instanceof ExcelPoiBasedStreamingStyleCustomizerInterface poiExcelStylerInterface)
+      {
+         this.styleCustomizerInterface = poiExcelStylerInterface;
+      }
+      else
+      {
+         LOG.debug("Supplied export style customizer is not an instance of ExcelPoiStyleCustomizerInterface, so will not be used for an excel export", logPair("exportStyleCustomizerClass", exportStyleCustomizer.getClass().getSimpleName()));
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingStyleCustomizerInterface.java

@@ -0,0 +1,81 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
+
+
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.reporting.ExportStyleCustomizerInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
+import org.apache.poi.ss.usermodel.CreationHelper;
+import org.apache.poi.xssf.usermodel.XSSFCellStyle;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+
+/*******************************************************************************
+ ** style customization points for Excel files generated via our streaming POI.
+ *******************************************************************************/
+public interface ExcelPoiBasedStreamingStyleCustomizerInterface extends ExportStyleCustomizerInterface
+{
+   /***************************************************************************
+    ** slightly legacy way we did excel styles - but get an instance of object
+    ** that defaults "default" styles (header, footer, etc).
+    ***************************************************************************/
+   default PoiExcelStylerInterface getExcelStyler()
+   {
+      return (new PlainPoiExcelStyler());
+   }
+
+
+   /***************************************************************************
+    ** either change "default" styles put in the styles map, or create new ones
+    ** which can then be applied to row/field values (cells) via:
+    ** ExcelPoiBasedStreamingExportStreamer.setStyleForField(row, fieldName, styleName);
+    ***************************************************************************/
+   default void customizeStyles(Map<String, XSSFCellStyle> styles, XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      //////////////////
+      // noop default //
+      //////////////////
+   }
+
+
+   /***************************************************************************
+    ** for a given view (sheet), return a list of custom column widths.
+    ** any nulls in the list are ignored (so default width is used).
+    ***************************************************************************/
+   default List<Integer> getColumnWidthsForView(QReportView view)
+   {
+      return (null);
+   }
+
+
+   /***************************************************************************
+    ** for a given view (sheet), return a list of any ranges which should be
+    ** merged, as in "A1:C1" (first three cells in first row).
+    ***************************************************************************/
+   default List<String> getMergedRangesForView(QReportView view)
+   {
+      return (null);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedSheetWriter.java

@@ -25,7 +25,10 @@ package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
 import java.io.IOException;
 import java.io.Writer;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import org.apache.poi.ss.util.CellReference;
 
 
@@ -53,13 +56,33 @@ public class StreamedSheetWriter
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void beginSheet() throws IOException
+   public void beginSheet(QReportView view, ExcelPoiBasedStreamingStyleCustomizerInterface styleCustomizerInterface) throws IOException
    {
       writer.write("""
          <?xml version="1.0" encoding="UTF-8"?>
-            <worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
-               <sheetData>""");
+            <worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">""");
 
+      if(styleCustomizerInterface != null && view != null)
+      {
+         List<Integer> columnWidths = styleCustomizerInterface.getColumnWidthsForView(view);
+         if(CollectionUtils.nullSafeHasContents(columnWidths))
+         {
+            writer.write("<cols>");
+            for(int i = 0; i < columnWidths.size(); i++)
+            {
+               Integer width = columnWidths.get(i);
+               if(width != null)
+               {
+                  writer.write("""
+                     <col min="%d" max="%d" width="%d" customWidth="1"/>
+                     """.formatted(i + 1, i + 1, width));
+               }
+            }
+            writer.write("</cols>");
+         }
+      }
+
+      writer.write("<sheetData>");
    }
 
 
@@ -67,11 +90,25 @@ public class StreamedSheetWriter
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void endSheet() throws IOException
+   public void endSheet(QReportView view, ExcelPoiBasedStreamingStyleCustomizerInterface styleCustomizerInterface) throws IOException
    {
-      writer.write("""
-            </sheetData>
-         </worksheet>""");
+      writer.write("</sheetData>");
+
+      if(styleCustomizerInterface != null && view != null)
+      {
+         List<String> mergedRanges = styleCustomizerInterface.getMergedRangesForView(view);
+         if(CollectionUtils.nullSafeHasContents(mergedRanges))
+         {
+            writer.write(String.format("<mergeCells count=\"%d\">", mergedRanges.size()));
+            for(String range : mergedRanges)
+            {
+               writer.write(String.format("<mergeCell ref=\"%s\"/>", range));
+            }
+            writer.write("</mergeCells>");
+         }
+      }
+
+      writer.write("</worksheet>");
    }
 
 
@@ -151,7 +188,7 @@ public class StreamedSheetWriter
             {
                rs.append("&quot;");
             }
-            else if (c < 32 && c != '\t' && c != '\n')
+            else if(c < 32 && c != '\t' && c != '\n')
             {
                rs.append(' ');
             }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/QJavaExecutor.java

@@ -53,7 +53,8 @@ public class QJavaExecutor implements QCodeExecutor
       Serializable output;
       try
       {
-         Function<Map<String, Object>, Serializable> function = QCodeLoader.getFunction(codeReference);
+         @SuppressWarnings("unchecked")
+         Function<Map<String, Object>, Serializable> function = QCodeLoader.getAdHoc(Function.class, codeReference);
          output = function.apply(context);
       }
       catch(Exception e)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfAction.java

@@ -26,22 +26,32 @@ import java.nio.file.Path;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.templates.ConvertHtmlToPdfInput;
 import com.kingsrook.qqq.backend.core.model.actions.templates.ConvertHtmlToPdfOutput;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.openhtmltopdf.css.constants.IdentValue;
+import com.openhtmltopdf.pdfboxout.PdfBoxFontResolver;
+import com.openhtmltopdf.pdfboxout.PdfBoxRenderer;
+import com.openhtmltopdf.pdfboxout.PdfRendererBuilder;
 import org.jsoup.Jsoup;
+import org.jsoup.helper.W3CDom;
 import org.jsoup.nodes.Document;
-import org.xhtmlrenderer.layout.SharedContext;
-import org.xhtmlrenderer.pdf.ITextRenderer;
 
 
 /*******************************************************************************
  ** Action to convert a string of HTML to a PDF!
  **
  ** Much credit to https://www.baeldung.com/java-html-to-pdf
- *******************************************************************************/
+ **
+ ** Updated in March 2025 to go from flying-saucer-pdf-openpdf lib to openhtmltopdf,
+ ** mostly to get support for max-height on images...
+ ********************************************************************************/
 public class ConvertHtmlToPdfAction extends AbstractQActionFunction<ConvertHtmlToPdfInput, ConvertHtmlToPdfOutput>
 {
+   private static final QLogger LOG = QLogger.getLogger(ConvertHtmlToPdfAction.class);
+
+
 
    /*******************************************************************************
     **
@@ -58,35 +68,37 @@ public class ConvertHtmlToPdfAction extends AbstractQActionFunction<ConvertHtmlT
          //////////////////////////////////////////////////////////////////
          Document document = Jsoup.parse(input.getHtml());
          document.outputSettings().syntax(Document.OutputSettings.Syntax.xml);
+         org.w3c.dom.Document w3cDoc = new W3CDom().fromJsoup(document);
 
          //////////////////////////////
          // convert the XHTML to PDF //
          //////////////////////////////
-         ITextRenderer renderer      = new ITextRenderer();
-         SharedContext sharedContext = renderer.getSharedContext();
-         sharedContext.setPrint(true);
-         sharedContext.setInteractive(false);
+         PdfRendererBuilder builder = new PdfRendererBuilder();
+         builder.toStream(input.getOutputStream());
+         builder.useFastMode();
+         builder.withW3cDocument(w3cDoc, input.getBasePath() == null ? "./" : input.getBasePath().toUri().toString());
 
-         if(input.getBasePath() != null)
+         try(PdfBoxRenderer pdfBoxRenderer = builder.buildPdfRenderer())
          {
-            String baseUrl = input.getBasePath().toUri().toURL().toString();
-            renderer.setDocumentFromString(document.html(), baseUrl);
-         }
-         else
-         {
-            renderer.setDocumentFromString(document.html());
-         }
+            pdfBoxRenderer.layout();
+            pdfBoxRenderer.getSharedContext().setPrint(true);
+            pdfBoxRenderer.getSharedContext().setInteractive(false);
 
-         //////////////////////////////////////////////////
-         // register any custom fonts the input supplied //
-         //////////////////////////////////////////////////
-         for(Map.Entry<String, Path> entry : CollectionUtils.nonNullMap(input.getCustomFonts()).entrySet())
-         {
-            renderer.getFontResolver().addFont(entry.getValue().toAbsolutePath().toString(), entry.getKey(), "UTF-8", true, null);
-         }
+            for(Map.Entry<String, Path> entry : CollectionUtils.nonNullMap(input.getCustomFonts()).entrySet())
+            {
+               LOG.warn("Note:  Custom fonts appear to not be working in this class at this time...");
+               pdfBoxRenderer.getFontResolver().addFont(
+                  entry.getValue().toAbsolutePath().toFile(), // Path to the TrueType font file
+                  entry.getKey(),                             // Font family name to use in CSS
+                  400,                                        // Font weight (e.g., 400 for normal, 700 for bold)
+                  IdentValue.NORMAL,                          // Font style (e.g., NORMAL, ITALIC)
+                  true,                                       // Whether to subset the font
+                  PdfBoxFontResolver.FontGroup.MAIN           // ??
+               );
+            }
 
-         renderer.layout();
-         renderer.createPDF(input.getOutputStream());
+            pdfBoxRenderer.createPDF();
+         }
 
          return (output);
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/BasicCustomPossibleValueProvider.java

@@ -0,0 +1,91 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.values;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
+
+
+/*******************************************************************************
+ ** Basic implementation of a possible value provider, for where there's a limited
+ ** set of possible source objects - so you just have to define how to make one
+ ** PV from a source object, how to list all of the source objects, and how to
+ ** look up a PV from an id.
+ *******************************************************************************/
+public abstract class BasicCustomPossibleValueProvider<S, ID extends Serializable> implements QCustomPossibleValueProvider<ID>
+{
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected abstract QPossibleValue<ID> makePossibleValue(S sourceObject);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected abstract S getSourceObject(Serializable id);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected abstract List<S> getAllSourceObjects();
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public QPossibleValue<ID> getPossibleValue(Serializable idValue)
+   {
+      S sourceObject = getSourceObject(idValue);
+      if(sourceObject == null)
+      {
+         return (null);
+      }
+
+      return makePossibleValue(sourceObject);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public List<QPossibleValue<ID>> search(SearchPossibleValueSourceInput input) throws QException
+   {
+      List<QPossibleValue<ID>> allPossibleValues = new ArrayList<>();
+      List<S>                  allSourceObjects  = getAllSourceObjects();
+      for(S sourceObject : allSourceObjects)
+      {
+         allPossibleValues.add(makePossibleValue(sourceObject));
+      }
+
+      return completeCustomPVSSearch(input, allPossibleValues);
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java

@@ -341,7 +341,7 @@ public class QPossibleValueTranslator
 
       try
       {
-         QCustomPossibleValueProvider customPossibleValueProvider = QCodeLoader.getCustomPossibleValueProvider(possibleValueSource);
+         QCustomPossibleValueProvider<?> customPossibleValueProvider = QCodeLoader.getAdHoc(QCustomPossibleValueProvider.class, possibleValueSource.getCustomCodeReference());
          return (formatPossibleValue(possibleValueSource, customPossibleValueProvider.getPossibleValue(value)));
       }
       catch(Exception e)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java

@@ -424,7 +424,7 @@ public class SearchPossibleValueSourceAction
    {
       try
       {
-         QCustomPossibleValueProvider customPossibleValueProvider = QCodeLoader.getCustomPossibleValueProvider(possibleValueSource);
+         QCustomPossibleValueProvider customPossibleValueProvider = QCodeLoader.getAdHoc(QCustomPossibleValueProvider.class, possibleValueSource.getCustomCodeReference());
          List<QPossibleValue<?>>      possibleValues              = customPossibleValueProvider.search(input);
 
          SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceOutput();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -1425,7 +1425,7 @@ public class QInstanceEnricher
          {
             try
             {
-               QCustomPossibleValueProvider<?> customPossibleValueProvider = QCodeLoader.getCustomPossibleValueProvider(possibleValueSource);
+               QCustomPossibleValueProvider<?> customPossibleValueProvider = QCodeLoader.getAdHoc(QCustomPossibleValueProvider.class, possibleValueSource.getCustomCodeReference());
 
                Method getPossibleValueMethod = customPossibleValueProvider.getClass().getDeclaredMethod("getPossibleValue", Serializable.class);
                Type   returnType             = getPossibleValueMethod.getGenericReturnType();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -45,6 +45,7 @@ import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph;
+import com.kingsrook.qqq.backend.core.actions.metadata.MetaDataActionCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.metadata.MetaDataFilterInterface;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
@@ -245,6 +246,11 @@ public class QInstanceValidator
       {
          validateSimpleCodeReference("Instance metaDataFilter ", qInstance.getMetaDataFilter(), MetaDataFilterInterface.class);
       }
+
+      if(qInstance.getMetaDataActionCustomizer() != null)
+      {
+         validateSimpleCodeReference("Instance metaDataActionCustomizer ", qInstance.getMetaDataActionCustomizer(), MetaDataActionCustomizerInterface.class);
+      }
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java

@@ -28,6 +28,7 @@ import java.util.Map;
 import java.util.function.Supplier;
 import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
 
 
@@ -44,6 +45,7 @@ public class ReportInput extends AbstractTableActionInput
    private ReportDestination reportDestination;
 
    private Supplier<? extends ExportStreamerInterface> overrideExportStreamerSupplier;
+   private QCodeReference                              exportStyleCustomizer;
 
 
 
@@ -208,4 +210,35 @@ public class ReportInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for exportStyleCustomizer
+    *******************************************************************************/
+   public QCodeReference getExportStyleCustomizer()
+   {
+      return (this.exportStyleCustomizer);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for exportStyleCustomizer
+    *******************************************************************************/
+   public void setExportStyleCustomizer(QCodeReference exportStyleCustomizer)
+   {
+      this.exportStyleCustomizer = exportStyleCustomizer;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for exportStyleCustomizer
+    *******************************************************************************/
+   public ReportInput withExportStyleCustomizer(QCodeReference exportStyleCustomizer)
+   {
+      this.exportStyleCustomizer = exportStyleCustomizer;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java

@@ -66,6 +66,7 @@ public enum WidgetType
    // record view/edit widgets //
    //////////////////////////////
    CHILD_RECORD_LIST("childRecordList"),
+   CUSTOM_COMPONENT("customComponent"),
    DYNAMIC_FORM("dynamicForm"),
    DATA_BAG_VIEWER("dataBagViewer"),
    PIVOT_TABLE_SETUP("pivotTableSetup"),

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java

@@ -468,7 +468,7 @@ public class QRecord implements Serializable
     *******************************************************************************/
    public String getValueString(String fieldName)
    {
-      return (ValueUtils.getValueAsString(values.get(fieldName)));
+      return (ValueUtils.getValueAsString(getValue(fieldName)));
    }
 
 
@@ -479,7 +479,7 @@ public class QRecord implements Serializable
     *******************************************************************************/
    public Integer getValueInteger(String fieldName)
    {
-      return (ValueUtils.getValueAsInteger(values.get(fieldName)));
+      return (ValueUtils.getValueAsInteger(getValue(fieldName)));
    }
 
 
@@ -490,7 +490,7 @@ public class QRecord implements Serializable
     *******************************************************************************/
    public Long getValueLong(String fieldName)
    {
-      return (ValueUtils.getValueAsLong(values.get(fieldName)));
+      return (ValueUtils.getValueAsLong(getValue(fieldName)));
    }
 
 
@@ -500,7 +500,7 @@ public class QRecord implements Serializable
     *******************************************************************************/
    public BigDecimal getValueBigDecimal(String fieldName)
    {
-      return (ValueUtils.getValueAsBigDecimal(values.get(fieldName)));
+      return (ValueUtils.getValueAsBigDecimal(getValue(fieldName)));
    }
 
 
@@ -510,7 +510,7 @@ public class QRecord implements Serializable
     *******************************************************************************/
    public Boolean getValueBoolean(String fieldName)
    {
-      return (ValueUtils.getValueAsBoolean(values.get(fieldName)));
+      return (ValueUtils.getValueAsBoolean(getValue(fieldName)));
    }
 
 
@@ -520,7 +520,7 @@ public class QRecord implements Serializable
     *******************************************************************************/
    public LocalTime getValueLocalTime(String fieldName)
    {
-      return (ValueUtils.getValueAsLocalTime(values.get(fieldName)));
+      return (ValueUtils.getValueAsLocalTime(getValue(fieldName)));
    }
 
 
@@ -530,7 +530,7 @@ public class QRecord implements Serializable
     *******************************************************************************/
    public LocalDate getValueLocalDate(String fieldName)
    {
-      return (ValueUtils.getValueAsLocalDate(values.get(fieldName)));
+      return (ValueUtils.getValueAsLocalDate(getValue(fieldName)));
    }
 
 
@@ -540,7 +540,7 @@ public class QRecord implements Serializable
     *******************************************************************************/
    public byte[] getValueByteArray(String fieldName)
    {
-      return (ValueUtils.getValueAsByteArray(values.get(fieldName)));
+      return (ValueUtils.getValueAsByteArray(getValue(fieldName)));
    }
 
 
@@ -550,7 +550,7 @@ public class QRecord implements Serializable
     *******************************************************************************/
    public Instant getValueInstant(String fieldName)
    {
-      return (ValueUtils.getValueAsInstant(values.get(fieldName)));
+      return (ValueUtils.getValueAsInstant(getValue(fieldName)));
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java

@@ -583,4 +583,31 @@ public abstract class QRecordEntity
       return (null);
    }
 
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static String getTableName(Class<? extends QRecordEntity> entityClass) throws QException
+   {
+      try
+      {
+         Field  tableNameField = entityClass.getDeclaredField("TABLE_NAME");
+         String tableNameValue = (String) tableNameField.get(null);
+         return (tableNameValue);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Could not get TABLE_NAME from entity class: " + entityClass.getSimpleName(), e));
+      }
+   }
+
+
+   /***************************************************************************
+    ** named without the 'get' to avoid conflict w/ entity fields named that...
+    ***************************************************************************/
+   public String tableName() throws QException
+   {
+      return (getTableName(this.getClass()));
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordWithJoinedRecords.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.data;
 
 import java.io.Serializable;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.function.BiFunction;
 
@@ -198,4 +199,62 @@ public class QRecordWithJoinedRecords extends QRecord
       return (rs);
    }
 
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public Map<String, List<QRecord>> getAssociatedRecords()
+   {
+      return mainRecord.getAssociatedRecords();
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public QRecord withAssociatedRecord(String name, QRecord associatedRecord)
+   {
+      mainRecord.withAssociatedRecord(name, associatedRecord);
+      return (this);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public QRecord withAssociatedRecords(Map<String, List<QRecord>> associatedRecords)
+   {
+      mainRecord.withAssociatedRecords(associatedRecords);
+      return (this);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void setAssociatedRecords(Map<String, List<QRecord>> associatedRecords)
+   {
+      mainRecord.setAssociatedRecords(associatedRecords);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public QRecord withAssociatedRecords(String name, List<QRecord> associatedRecords)
+   {
+      mainRecord.withAssociatedRecords(name, associatedRecords);
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/helpcontent/HelpContentMetaDataProvider.java

@@ -82,6 +82,7 @@ public class HelpContentMetaDataProvider
       table.getField("key").withFieldAdornment(AdornmentType.Size.LARGE.toAdornment());
       table.getField("content").withFieldAdornment(AdornmentType.Size.LARGE.toAdornment());
       table.getField("content").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("html")));
+      table.getField("content").withGridColumns(12);
 
       if(backendDetailEnricher != null)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java

@@ -417,7 +417,7 @@ public class MetaDataProducerHelper
          return (null);
       }
 
-      ChildJoinFromRecordEntityGenericMetaDataProducer producer = new ChildJoinFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, possibleValueFieldName);
+      ChildJoinFromRecordEntityGenericMetaDataProducer producer = new ChildJoinFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, possibleValueFieldName, childTable.childJoin().orderBy());
       producer.setSourceClass(entityClass);
       return producer;
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java

@@ -116,8 +116,11 @@ public class QInstance
    private QPermissionRules defaultPermissionRules = QPermissionRules.defaultInstance();
    private QAuditRules      defaultAuditRules      = QAuditRules.defaultInstanceLevelNone();
 
+   @Deprecated(since = "migrated to metaDataCustomizer")
    private QCodeReference metaDataFilter = null;
 
+   private QCodeReference metaDataActionCustomizer = null;
+
    //////////////////////////////////////////////////////////////////////////////////////
    // todo - lock down the object (no more changes allowed) after it's been validated? //
    //  if doing so, may need to copy all of the collections into read-only versions... //
@@ -1495,6 +1498,7 @@ public class QInstance
    /*******************************************************************************
     ** Getter for metaDataFilter
     *******************************************************************************/
+   @Deprecated(since = "migrated to metaDataCustomizer")
    public QCodeReference getMetaDataFilter()
    {
       return (this.metaDataFilter);
@@ -1505,6 +1509,7 @@ public class QInstance
    /*******************************************************************************
     ** Setter for metaDataFilter
     *******************************************************************************/
+   @Deprecated(since = "migrated to metaDataCustomizer")
    public void setMetaDataFilter(QCodeReference metaDataFilter)
    {
       this.metaDataFilter = metaDataFilter;
@@ -1515,6 +1520,7 @@ public class QInstance
    /*******************************************************************************
     ** Fluent setter for metaDataFilter
     *******************************************************************************/
+   @Deprecated(since = "migrated to metaDataCustomizer")
    public QInstance withMetaDataFilter(QCodeReference metaDataFilter)
    {
       this.metaDataFilter = metaDataFilter;
@@ -1586,4 +1592,35 @@ public class QInstance
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for metaDataActionCustomizer
+    *******************************************************************************/
+   public QCodeReference getMetaDataActionCustomizer()
+   {
+      return (this.metaDataActionCustomizer);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for metaDataActionCustomizer
+    *******************************************************************************/
+   public void setMetaDataActionCustomizer(QCodeReference metaDataActionCustomizer)
+   {
+      this.metaDataActionCustomizer = metaDataActionCustomizer;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for metaDataActionCustomizer
+    *******************************************************************************/
+   public QInstance withMetaDataActionCustomizer(QCodeReference metaDataActionCustomizer)
+   {
+      this.metaDataActionCustomizer = metaDataActionCustomizer;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/Banner.java

@@ -0,0 +1,269 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.branding;
+
+
+import java.io.Serializable;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+
+/*******************************************************************************
+ ** element of BrandingMetaData - content to send to a frontend for showing a
+ ** user across the whole UI - e.g., what environment you're in, or a message
+ ** about your account - site announcements, etc.  
+ *******************************************************************************/
+public class Banner implements Serializable, Cloneable
+{
+   private Severity severity;
+   private String   textColor;
+   private String   backgroundColor;
+   private String   messageText;
+   private String   messageHTML;
+
+   private Map<String, Serializable> additionalStyles;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public enum Severity
+   {
+      INFO, WARNING, ERROR, SUCCESS
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public Banner clone()
+   {
+      try
+      {
+         Banner clone = (Banner) super.clone();
+
+         //////////////////////////////////////////////////////////////////////////////////////
+         // copy mutable state here, so the clone can't change the internals of the original //
+         //////////////////////////////////////////////////////////////////////////////////////
+         if(additionalStyles != null)
+         {
+            clone.setAdditionalStyles(new LinkedHashMap<>(additionalStyles));
+         }
+
+         return clone;
+      }
+      catch(CloneNotSupportedException e)
+      {
+         throw new AssertionError();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for textColor
+    *******************************************************************************/
+   public String getTextColor()
+   {
+      return (this.textColor);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for textColor
+    *******************************************************************************/
+   public void setTextColor(String textColor)
+   {
+      this.textColor = textColor;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for textColor
+    *******************************************************************************/
+   public Banner withTextColor(String textColor)
+   {
+      this.textColor = textColor;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for backgroundColor
+    *******************************************************************************/
+   public String getBackgroundColor()
+   {
+      return (this.backgroundColor);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for backgroundColor
+    *******************************************************************************/
+   public void setBackgroundColor(String backgroundColor)
+   {
+      this.backgroundColor = backgroundColor;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for backgroundColor
+    *******************************************************************************/
+   public Banner withBackgroundColor(String backgroundColor)
+   {
+      this.backgroundColor = backgroundColor;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for additionalStyles
+    *******************************************************************************/
+   public Map<String, Serializable> getAdditionalStyles()
+   {
+      return (this.additionalStyles);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for additionalStyles
+    *******************************************************************************/
+   public void setAdditionalStyles(Map<String, Serializable> additionalStyles)
+   {
+      this.additionalStyles = additionalStyles;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for additionalStyles
+    *******************************************************************************/
+   public Banner withAdditionalStyles(Map<String, Serializable> additionalStyles)
+   {
+      this.additionalStyles = additionalStyles;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for messageText
+    *******************************************************************************/
+   public String getMessageText()
+   {
+      return (this.messageText);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for messageText
+    *******************************************************************************/
+   public void setMessageText(String messageText)
+   {
+      this.messageText = messageText;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for messageText
+    *******************************************************************************/
+   public Banner withMessageText(String messageText)
+   {
+      this.messageText = messageText;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for messageHTML
+    *******************************************************************************/
+   public String getMessageHTML()
+   {
+      return (this.messageHTML);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for messageHTML
+    *******************************************************************************/
+   public void setMessageHTML(String messageHTML)
+   {
+      this.messageHTML = messageHTML;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for messageHTML
+    *******************************************************************************/
+   public Banner withMessageHTML(String messageHTML)
+   {
+      this.messageHTML = messageHTML;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for severity
+    *******************************************************************************/
+   public Severity getSeverity()
+   {
+      return (this.severity);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for severity
+    *******************************************************************************/
+   public void setSeverity(Severity severity)
+   {
+      this.severity = severity;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for severity
+    *******************************************************************************/
+   public Banner withSeverity(Severity severity)
+   {
+      this.severity = severity;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/BannerSlot.java

@@ -0,0 +1,31 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.branding;
+
+
+/*******************************************************************************
+ ** interface to define keys for where banners should be displayed.
+ ** expect frontends to implement this interface with enums of known possible values
+ *******************************************************************************/
+public interface BannerSlot
+{
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java

@@ -22,6 +22,9 @@
 package com.kingsrook.qqq.backend.core.model.metadata.branding;
 
 
+import java.io.Serializable;
+import java.util.LinkedHashMap;
+import java.util.Map;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
 
@@ -30,7 +33,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
  ** Meta-Data to define branding in a QQQ instance.
  **
  *******************************************************************************/
-public class QBrandingMetaData implements TopLevelMetaDataInterface
+public class QBrandingMetaData implements TopLevelMetaDataInterface, Cloneable, Serializable
 {
    private String companyName;
    private String companyUrl;
@@ -39,9 +42,45 @@ public class QBrandingMetaData implements TopLevelMetaDataInterface
    private String icon;
    private String accentColor;
 
+   @Deprecated(since = "migrate to use banners map instead")
    private String environmentBannerText;
+
+   @Deprecated(since = "migrate to use banners map instead")
    private String environmentBannerColor;
 
+   private Map<BannerSlot, Banner> banners;
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public QBrandingMetaData clone()
+   {
+      try
+      {
+         QBrandingMetaData clone = (QBrandingMetaData) super.clone();
+
+         //////////////////////////////////////////////////////////////////////////////////////
+         // copy mutable state here, so the clone can't change the internals of the original //
+         //////////////////////////////////////////////////////////////////////////////////////
+         if(banners != null)
+         {
+            clone.banners = new LinkedHashMap<>();
+            for(Map.Entry<BannerSlot, Banner> entry : this.banners.entrySet())
+            {
+               clone.banners.put(entry.getKey(), entry.getValue().clone());
+            }
+         }
+
+         return clone;
+      }
+      catch(CloneNotSupportedException e)
+      {
+         throw new AssertionError();
+      }
+   }
+
 
 
    /*******************************************************************************
@@ -267,6 +306,7 @@ public class QBrandingMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Getter for environmentBannerText
     *******************************************************************************/
+   @Deprecated(since = "migrate to use banners map instead")
    public String getEnvironmentBannerText()
    {
       return (this.environmentBannerText);
@@ -277,6 +317,7 @@ public class QBrandingMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Setter for environmentBannerText
     *******************************************************************************/
+   @Deprecated(since = "migrate to use banners map instead")
    public void setEnvironmentBannerText(String environmentBannerText)
    {
       this.environmentBannerText = environmentBannerText;
@@ -287,6 +328,7 @@ public class QBrandingMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for environmentBannerText
     *******************************************************************************/
+   @Deprecated(since = "migrate to use banners map instead")
    public QBrandingMetaData withEnvironmentBannerText(String environmentBannerText)
    {
       this.environmentBannerText = environmentBannerText;
@@ -298,6 +340,7 @@ public class QBrandingMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Getter for environmentBannerColor
     *******************************************************************************/
+   @Deprecated(since = "migrate to use banners map instead")
    public String getEnvironmentBannerColor()
    {
       return (this.environmentBannerColor);
@@ -308,6 +351,7 @@ public class QBrandingMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Setter for environmentBannerColor
     *******************************************************************************/
+   @Deprecated(since = "migrate to use banners map instead")
    public void setEnvironmentBannerColor(String environmentBannerColor)
    {
       this.environmentBannerColor = environmentBannerColor;
@@ -318,6 +362,7 @@ public class QBrandingMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for environmentBannerColor
     *******************************************************************************/
+   @Deprecated(since = "migrate to use banners map instead")
    public QBrandingMetaData withEnvironmentBannerColor(String environmentBannerColor)
    {
       this.environmentBannerColor = environmentBannerColor;
@@ -334,4 +379,52 @@ public class QBrandingMetaData implements TopLevelMetaDataInterface
    {
       qInstance.setBranding(this);
    }
+
+
+   /*******************************************************************************
+    ** Getter for banners
+    *******************************************************************************/
+   public Map<BannerSlot, Banner> getBanners()
+   {
+      return (this.banners);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for banners
+    *******************************************************************************/
+   public void setBanners(Map<BannerSlot, Banner> banners)
+   {
+      this.banners = banners;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for banners
+    *******************************************************************************/
+   public QBrandingMetaData withBanners(Map<BannerSlot, Banner> banners)
+   {
+      this.banners = banners;
+      return (this);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public QBrandingMetaData withBanner(BannerSlot slot, Banner banner)
+   {
+      if(this.banners == null)
+      {
+         this.banners = new LinkedHashMap<>();
+      }
+      this.banners.put(slot, banner);
+
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/InitializableViaCodeReference.java

@@ -0,0 +1,38 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.code;
+
+
+/*******************************************************************************
+ ** an object which is intended to be constructed via a CodeReference, and,
+ ** moreso, after it is created, then the initialize method here gets called,
+ ** passing the codeRefernce in - e.g., to do additional initalization of the
+ ** object, e.g., properties in a QCodeReferenceWithProperties
+ *******************************************************************************/
+public interface InitializableViaCodeReference
+{
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   void initialize(QCodeReference codeReference);
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeReference.java

@@ -29,7 +29,7 @@ import java.io.Serializable;
  ** Pointer to code to be ran by the qqq framework, e.g., for custom behavior -
  ** maybe process steps, maybe customization to a table, etc.
  *******************************************************************************/
-public class QCodeReference implements Serializable
+public class QCodeReference implements Serializable, Cloneable
 {
    private String    name;
    private QCodeType codeType;
@@ -58,6 +58,25 @@ public class QCodeReference implements Serializable
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public QCodeReference clone()
+   {
+      try
+      {
+         QCodeReference clone = (QCodeReference) super.clone();
+         return clone;
+      }
+      catch(CloneNotSupportedException e)
+      {
+         throw new AssertionError();
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -179,5 +198,4 @@ public class QCodeReference implements Serializable
       this.inlineCode = inlineCode;
       return (this);
    }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeReferenceWithProperties.java

@@ -0,0 +1,59 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.code;
+
+
+import java.io.Serializable;
+import java.util.Map;
+
+
+/*******************************************************************************
+ ** a code reference that also has a map of properties.  This object (with the
+ ** properties) will be passed in to the referenced object, if it implements
+ ** InitializableViaCodeReference.
+ *******************************************************************************/
+public class QCodeReferenceWithProperties extends QCodeReference
+{
+   private final Map<String, Serializable> properties;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public QCodeReferenceWithProperties(Class<?> javaClass, Map<String, Serializable> properties)
+   {
+      super(javaClass);
+      this.properties = properties;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for properties
+    **
+    *******************************************************************************/
+   public Map<String, Serializable> getProperties()
+   {
+      return properties;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/WidgetAdHocValue.java

@@ -62,7 +62,8 @@ public class WidgetAdHocValue extends AbstractWidgetValueSource
          context.putAll(inputValues);
       }
 
-      Function<Object, Object> function = QCodeLoader.getFunction(codeReference);
+      @SuppressWarnings("unchecked")
+      Function<Object, Object> function = QCodeLoader.getAdHoc(Function.class, codeReference);
       Object                   result   = function.apply(context);
       return (result);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java

@@ -238,7 +238,7 @@ public class QFieldMetaData implements Cloneable
 
             if(StringUtils.hasContent(fieldAnnotation.defaultValue()))
             {
-               ValueUtils.getValueAsFieldType(this.type, fieldAnnotation.defaultValue());
+               withDefaultValue(ValueUtils.getValueAsFieldType(this.type, fieldAnnotation.defaultValue()));
             }
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/WhiteSpaceBehavior.java

@@ -0,0 +1,174 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.fields;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ ** Field behavior that changes the whitespace of string values.
+ *******************************************************************************/
+public enum WhiteSpaceBehavior implements FieldBehavior<WhiteSpaceBehavior>, FieldBehaviorForFrontend, FieldFilterBehavior<WhiteSpaceBehavior>
+{
+   NONE(null),
+   REMOVE_ALL_WHITESPACE((String s) -> s.chars().filter(c -> !Character.isWhitespace(c)).collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append).toString()),
+   TRIM((String s) -> s.trim()),
+   TRIM_LEFT((String s) -> s.stripLeading()),
+   TRIM_RIGHT((String s) -> s.stripTrailing());
+
+
+   private final Function<String, String> function;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   WhiteSpaceBehavior(Function<String, String> function)
+   {
+      this.function = function;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public WhiteSpaceBehavior getDefault()
+   {
+      return (NONE);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
+   {
+      if(this.equals(NONE))
+      {
+         return;
+      }
+
+      switch(this)
+      {
+         case REMOVE_ALL_WHITESPACE, TRIM, TRIM_LEFT, TRIM_RIGHT -> applyFunction(recordList, table, field);
+         default -> throw new IllegalStateException("Unexpected enum value: " + this);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void applyFunction(List<QRecord> recordList, QTableMetaData table, QFieldMetaData field)
+   {
+      String fieldName = field.getName();
+      for(QRecord record : CollectionUtils.nonNullList(recordList))
+      {
+         String value = record.getValueString(fieldName);
+         if(value != null && function != null)
+         {
+            record.setValue(fieldName, function.apply(value));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public Serializable applyToFilterCriteriaValue(Serializable value, QInstance instance, QTableMetaData table, QFieldMetaData field)
+   {
+      if(this.equals(NONE) || function == null)
+      {
+         return (value);
+      }
+
+      if(value instanceof String s)
+      {
+         String newValue = function.apply(s);
+         if(!Objects.equals(value, newValue))
+         {
+            return (newValue);
+         }
+      }
+
+      return (value);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean allowMultipleBehaviorsOfThisType()
+   {
+      return (false);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<String> validateBehaviorConfiguration(QTableMetaData tableMetaData, QFieldMetaData fieldMetaData)
+   {
+      if(this == NONE)
+      {
+         return Collections.emptyList();
+      }
+
+      List<String> errors      = new ArrayList<>();
+      String       errorSuffix = " field [" + fieldMetaData.getName() + "] in table [" + tableMetaData.getName() + "]";
+
+      if(fieldMetaData.getType() != null)
+      {
+         if(!fieldMetaData.getType().isStringLike())
+         {
+            errors.add("A WhiteSpaceBehavior was a applied to a non-String-like field:" + errorSuffix);
+         }
+      }
+
+      return (errors);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java

@@ -55,6 +55,7 @@ public class QFrontendFieldMetaData implements Serializable
    private String       possibleValueSourceName;
    private String       displayFormat;
    private Serializable defaultValue;
+   private Integer      maxLength;
 
    private List<FieldAdornment> adornments;
    private List<QHelpContent>   helpContents;
@@ -85,6 +86,7 @@ public class QFrontendFieldMetaData implements Serializable
       this.defaultValue = fieldMetaData.getDefaultValue();
       this.helpContents = fieldMetaData.getHelpContents();
       this.inlinePossibleValueSource = fieldMetaData.getInlinePossibleValueSource();
+      this.maxLength = fieldMetaData.getMaxLength();
 
       for(FieldBehavior<?> behavior : CollectionUtils.nonNullCollection(fieldMetaData.getBehaviors()))
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendProcessMetaData.java

@@ -48,6 +48,8 @@ public class QFrontendProcessMetaData
    private String  label;
    private String  tableName;
    private boolean isHidden;
+   private Integer minInputRecords;
+   private Integer maxInputRecords;
 
    private QIcon icon;
 
@@ -72,6 +74,8 @@ public class QFrontendProcessMetaData
       this.tableName = processMetaData.getTableName();
       this.isHidden = processMetaData.getIsHidden();
       this.stepFlow = processMetaData.getStepFlow().toString();
+      this.minInputRecords = processMetaData.getMinInputRecords();
+      this.maxInputRecords = processMetaData.getMaxInputRecords();
 
       if(includeSteps)
       {
@@ -213,4 +217,27 @@ public class QFrontendProcessMetaData
    {
       return icon;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for minInputRecords
+    **
+    *******************************************************************************/
+   public Integer getMinInputRecords()
+   {
+      return minInputRecords;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for maxInputRecords
+    **
+    *******************************************************************************/
+   public Integer getMaxInputRecords()
+   {
+      return maxInputRecords;
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QIcon.java

@@ -31,7 +31,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.layout;
  ** Future may allow something like a "namespace", and/or multiple icons for
  ** use in different frontends, etc.
  *******************************************************************************/
-public class QIcon
+public class QIcon implements Cloneable
 {
    private String name;
    private String path;
@@ -58,6 +58,25 @@ public class QIcon
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public QIcon clone()
+   {
+      try
+      {
+         QIcon clone = (QIcon) super.clone();
+         return clone;
+      }
+      catch(CloneNotSupportedException e)
+      {
+         throw new AssertionError();
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Getter for name
     **
@@ -154,6 +173,4 @@ public class QIcon
       this.color = color;
       return (this);
    }
-
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SendSESAction.java

@@ -216,11 +216,16 @@ public class SendSESAction
       {
          LOG.warn("More than one FROM value was found, will send using the first one found [" + partyList.get(0).getAddress() + "].");
       }
+      Party fromParty = partyList.get(0);
+      if(fromParty.getAddress() == null)
+      {
+         throw (new QException("Cannot send SES message because a FROM address was not provided."));
+      }
 
       /////////////////////////////
       // return the from address //
       /////////////////////////////
-      return (partyList.get(0).getAddress());
+      return (getFullEmailAddress(fromParty));
    }
 
 
@@ -267,15 +272,15 @@ public class SendSESAction
       {
          if(EmailPartyRole.CC.equals(party.getRole()))
          {
-            ccList.add(party.getAddress());
+            ccList.add(getFullEmailAddress(party));
          }
          else if(EmailPartyRole.BCC.equals(party.getRole()))
          {
-            bccList.add(party.getAddress());
+            bccList.add(getFullEmailAddress(party));
          }
          else if(party.getRole() == null || PartyRole.Default.DEFAULT.equals(party.getRole()) || EmailPartyRole.TO.equals(party.getRole()))
          {
-            toList.add(party.getAddress());
+            toList.add(getFullEmailAddress(party));
          }
          else
          {
@@ -332,4 +337,22 @@ public class SendSESAction
 
       return amazonSES;
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String getFullEmailAddress(Party party)
+   {
+      if(party.getLabel() != null)
+      {
+         return (party.getLabel() + " <" + party.getAddress() + ">");
+      }
+
+      /////////////////////////////
+      // return the from address //
+      /////////////////////////////
+      return (party.getAddress());
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildJoinFromRecordEntityGenericMetaDataProducer.java

@@ -24,11 +24,13 @@ package com.kingsrook.qqq.backend.core.model.metadata.producers;
 
 import java.util.Objects;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.annotations.ChildJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 
 
@@ -39,12 +41,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
  **
  ** e.g., Orders & LineItems - on the Order entity
  ** <code>
- @QMetaDataProducingEntity(
-    childTables = { @ChildTable(
-       childTableEntityClass = LineItem.class,
-       childJoin = @ChildJoin(enabled = true),
-       childRecordListWidget = @ChildRecordListWidget(enabled = true, label = "Order Lines"))
-    }
+ @QMetaDataProducingEntity( childTables = { @ChildTable(
+ childTableEntityClass = LineItem.class,
+ childJoin = @ChildJoin(enabled = true),
+ childRecordListWidget = @ChildRecordListWidget(enabled = true, label = "Order Lines"))
+ }
  )
  public class Order extends QRecordEntity
  ** </code>
@@ -62,13 +63,16 @@ public class ChildJoinFromRecordEntityGenericMetaDataProducer implements MetaDat
    private String parentTableName; // e.g., order
    private String foreignKeyFieldName; // e.g., orderId
 
+   private ChildJoin.OrderBy[] orderBys;
+
    private Class<?> sourceClass;
 
 
+
    /***************************************************************************
     **
     ***************************************************************************/
-   public ChildJoinFromRecordEntityGenericMetaDataProducer(String childTableName, String parentTableName, String foreignKeyFieldName)
+   public ChildJoinFromRecordEntityGenericMetaDataProducer(String childTableName, String parentTableName, String foreignKeyFieldName, ChildJoin.OrderBy[] orderBys)
    {
       Objects.requireNonNull(childTableName, "childTableName cannot be null");
       Objects.requireNonNull(parentTableName, "parentTableName cannot be null");
@@ -77,6 +81,7 @@ public class ChildJoinFromRecordEntityGenericMetaDataProducer implements MetaDat
       this.childTableName = childTableName;
       this.parentTableName = parentTableName;
       this.foreignKeyFieldName = foreignKeyFieldName;
+      this.orderBys = orderBys;
    }
 
 
@@ -87,18 +92,39 @@ public class ChildJoinFromRecordEntityGenericMetaDataProducer implements MetaDat
    @Override
    public QJoinMetaData produce(QInstance qInstance) throws QException
    {
-      QTableMetaData possibleValueTable = qInstance.getTable(parentTableName);
-      if(possibleValueTable == null)
+      QTableMetaData parentTable = qInstance.getTable(parentTableName);
+      if(parentTable == null)
       {
          throw (new QException("Could not find tableMetaData " + parentTableName));
       }
 
+      QTableMetaData childTable = qInstance.getTable(childTableName);
+      if(childTable == null)
+      {
+         throw (new QException("Could not find tableMetaData " + childTable));
+      }
+
       QJoinMetaData join = new QJoinMetaData()
          .withLeftTable(parentTableName)
          .withRightTable(childTableName)
          .withInferredName()
          .withType(JoinType.ONE_TO_MANY)
-         .withJoinOn(new JoinOn(possibleValueTable.getPrimaryKeyField(), foreignKeyFieldName));
+         .withJoinOn(new JoinOn(parentTable.getPrimaryKeyField(), foreignKeyFieldName));
+
+      if(orderBys != null && orderBys.length > 0)
+      {
+         for(ChildJoin.OrderBy orderBy : orderBys)
+         {
+            join.withOrderBy(new QFilterOrderBy(orderBy.fieldName(), orderBy.isAscending()));
+         }
+      }
+      else
+      {
+         //////////////////////////////////////////////////////////
+         // by default, sort by the id of the child table... mmm //
+         //////////////////////////////////////////////////////////
+         join.withOrderBy(new QFilterOrderBy(childTable.getPrimaryKeyField()));
+      }
 
       return (join);
    }
@@ -126,6 +152,7 @@ public class ChildJoinFromRecordEntityGenericMetaDataProducer implements MetaDat
    }
 
 
+
    /*******************************************************************************
     ** Fluent setter for sourceClass
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/annotations/ChildJoin.java

@@ -35,4 +35,16 @@ import java.lang.annotation.RetentionPolicy;
 public @interface ChildJoin
 {
    boolean enabled();
+
+   OrderBy[] orderBy() default { };
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @interface OrderBy
+   {
+      String fieldName();
+
+      boolean isAscending() default true;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportDataSource.java

@@ -40,7 +40,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
  ** (optionally along with queryJoins and queryInputCustomizer) is used.
  ** - else a staticDataSupplier is used.
  *******************************************************************************/
-public class QReportDataSource
+public class QReportDataSource implements Cloneable
 {
    private String name;
 
@@ -55,6 +55,39 @@ public class QReportDataSource
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public QReportDataSource clone()
+   {
+      try
+      {
+         QReportDataSource clone = (QReportDataSource) super.clone();
+         if(queryFilter != null)
+         {
+            clone.queryFilter = queryFilter.clone();
+         }
+
+         if(queryJoins != null)
+         {
+            clone.queryJoins = new ArrayList<>();
+            for(QueryJoin join : queryJoins)
+            {
+               queryJoins.add(join.clone());
+            }
+         }
+
+         return clone;
+      }
+      catch(CloneNotSupportedException e)
+      {
+         throw new AssertionError();
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Getter for name
     **
@@ -274,6 +307,7 @@ public class QReportDataSource
    }
 
 
+
    /*******************************************************************************
     ** Getter for customRecordSource
     *******************************************************************************/
@@ -303,5 +337,4 @@ public class QReportDataSource
       return (this);
    }
 
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportMetaData.java

@@ -26,6 +26,7 @@ import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
@@ -37,7 +38,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 /*******************************************************************************
  ** Meta-data definition of a report generated by QQQ
  *******************************************************************************/
-public class QReportMetaData implements QAppChildMetaData, MetaDataWithPermissionRules, TopLevelMetaDataInterface
+public class QReportMetaData implements QAppChildMetaData, MetaDataWithPermissionRules, TopLevelMetaDataInterface, Cloneable
 {
    private String name;
    private String label;
@@ -52,6 +53,72 @@ public class QReportMetaData implements QAppChildMetaData, MetaDataWithPermissio
 
    private QIcon icon;
 
+   private QCodeReference exportStyleCustomizer;
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public QReportMetaData clone()
+   {
+      try
+      {
+         QReportMetaData clone = (QReportMetaData) super.clone();
+
+         //////////////////////////////
+         // Deep copy mutable fields //
+         //////////////////////////////
+         if(this.inputFields != null)
+         {
+            clone.inputFields = new ArrayList<>();
+            for(QFieldMetaData inputField : this.inputFields)
+            {
+               clone.inputFields.add(inputField.clone());
+            }
+         }
+
+         if(this.dataSources != null)
+         {
+            clone.dataSources = new ArrayList<>();
+            for(QReportDataSource dataSource : this.dataSources)
+            {
+               clone.dataSources.add(dataSource.clone());
+            }
+         }
+
+         if(this.views != null)
+         {
+            clone.views = new ArrayList<>();
+            for(QReportView view : this.views)
+            {
+               clone.views.add(view.clone());
+            }
+         }
+
+         if(this.permissionRules != null)
+         {
+            clone.permissionRules = this.permissionRules.clone();
+         }
+
+         if(this.icon != null)
+         {
+            clone.icon = this.icon.clone();
+         }
+
+         if(this.exportStyleCustomizer != null)
+         {
+            clone.exportStyleCustomizer = this.exportStyleCustomizer.clone();
+         }
+
+         return clone;
+      }
+      catch(CloneNotSupportedException e)
+      {
+         throw new AssertionError("Cloning not supported", e);
+      }
+   }
+
 
 
    /*******************************************************************************
@@ -397,4 +464,35 @@ public class QReportMetaData implements QAppChildMetaData, MetaDataWithPermissio
       qInstance.addReport(this);
    }
 
+
+   /*******************************************************************************
+    ** Getter for exportStyleCustomizer
+    *******************************************************************************/
+   public QCodeReference getExportStyleCustomizer()
+   {
+      return (this.exportStyleCustomizer);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for exportStyleCustomizer
+    *******************************************************************************/
+   public void setExportStyleCustomizer(QCodeReference exportStyleCustomizer)
+   {
+      this.exportStyleCustomizer = exportStyleCustomizer;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for exportStyleCustomizer
+    *******************************************************************************/
+   public QReportMetaData withExportStyleCustomizer(QCodeReference exportStyleCustomizer)
+   {
+      this.exportStyleCustomizer = exportStyleCustomizer;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java

@@ -27,11 +27,9 @@ import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
-import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
+import com.kingsrook.qqq.backend.core.actions.values.BasicCustomPossibleValueProvider;
 import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
-import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
@@ -40,26 +38,16 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
  ** possible-value source provider for the `Tables` PVS - a list of all tables
  ** in an application/qInstance.
  *******************************************************************************/
-public class TablesCustomPossibleValueProvider implements QCustomPossibleValueProvider<String>
+public class TablesCustomPossibleValueProvider extends BasicCustomPossibleValueProvider<QTableMetaData, String>
 {
 
    /***************************************************************************
     **
     ***************************************************************************/
    @Override
-   public QPossibleValue<String> getPossibleValue(Serializable idValue)
+   protected QPossibleValue<String> makePossibleValue(QTableMetaData sourceObject)
    {
-      QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(idValue));
-      if(table != null && !table.getIsHidden())
-      {
-         PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table);
-         if(PermissionCheckResult.ALLOW.equals(permissionCheckResult))
-         {
-            return (new QPossibleValue<>(table.getName(), table.getLabel()));
-         }
-      }
-
-      return null;
+      return (new QPossibleValue<>(sourceObject.getName(), sourceObject.getLabel()));
    }
 
 
@@ -68,22 +56,54 @@ public class TablesCustomPossibleValueProvider implements QCustomPossibleValuePr
     **
     ***************************************************************************/
    @Override
-   public List<QPossibleValue<String>> search(SearchPossibleValueSourceInput input) throws QException
+   protected QTableMetaData getSourceObject(Serializable id)
    {
-      /////////////////////////////////////////////////////////////////////////////////////
-      // build all of the possible values (note, will be filtered by user's permissions) //
-      /////////////////////////////////////////////////////////////////////////////////////
-      List<QPossibleValue<String>> allPossibleValues = new ArrayList<>();
+      QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(id));
+      return isTableAllowed(table) ? table : null;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   protected List<QTableMetaData> getAllSourceObjects()
+   {
+      ArrayList<QTableMetaData> rs = new ArrayList<>();
       for(QTableMetaData table : QContext.getQInstance().getTables().values())
       {
-         QPossibleValue<String> possibleValue = getPossibleValue(table.getName());
-         if(possibleValue != null)
+         if(isTableAllowed(table))
          {
-            allPossibleValues.add(possibleValue);
+            rs.add(table);
          }
       }
+      return rs;
+   }
 
-      return completeCustomPVSSearch(input, allPossibleValues);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private boolean isTableAllowed(QTableMetaData table)
+   {
+      if(table == null)
+      {
+         return (false);
+      }
+
+      if(table.getIsHidden())
+      {
+         return (false);
+      }
+
+      PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table);
+      if(!PermissionCheckResult.ALLOW.equals(permissionCheckResult))
+      {
+         return (false);
+      }
+
+      return (true);
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables;
 
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
@@ -45,6 +46,7 @@ public class TablesPossibleValueSourceMetaDataProvider
    {
       QPossibleValueSource possibleValueSource = new QPossibleValueSource()
          .withName(NAME)
+         .withIdType(QFieldType.STRING)
          .withType(QPossibleValueSourceType.CUSTOM)
          .withCustomCodeReference(new QCodeReference(TablesCustomPossibleValueProvider.class))
          .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java

@@ -55,8 +55,6 @@ public class BulkInsertExtractStep extends AbstractExtractStep
    @Override
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
-      runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput));
-
       int rowsAdded     = 0;
       int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java

@@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
+import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
@@ -77,19 +78,77 @@ public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSumm
 
       QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getValueString("tableName"));
 
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // the transform step builds summary lines that it predicts will insert successfully.      //
+      // but those lines don't have ids, which we'd like to have (e.g., for a process trace that //
+      // might link to the built record).  also, it's possible that there was a fail that only   //
+      // happened in the actual insert, so, basically, re-do the summary here                    //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      BulkInsertTransformStep transformStep = (BulkInsertTransformStep) getTransformStep();
+      ProcessSummaryLine      okSummary     = transformStep.okSummary;
+      okSummary.setCount(0);
+      okSummary.setPrimaryKeys(new ArrayList<>());
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // but - since errors from the transform step don't even make it through to us here in the load step,   //
+      // do re-use the ProcessSummaryWarningsAndErrorsRollup from transform step as follows:                  //
+      // clear out its warnings - we'll completely rebuild them here (with primary keys)                      //
+      // and add new error lines, e.g., in case of errors that only happened past the validation if possible. //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////
+      ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = transformStep.processSummaryWarningsAndErrorsRollup;
+      processSummaryWarningsAndErrorsRollup.resetWarnings();
+
       List<QRecord> insertedRecords = runBackendStepOutput.getRecords();
       for(QRecord insertedRecord : insertedRecords)
       {
-         if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()))
+         Serializable primaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
+         if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()) && primaryKey != null)
          {
+            /////////////////////////////////////////////////////////////////////////
+            // if the record had no errors, and we have a primary key for it, then //
+            // keep track of the range of primary keys (first and last)            //
+            /////////////////////////////////////////////////////////////////////////
             if(firstInsertedPrimaryKey == null)
             {
-               firstInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
+               firstInsertedPrimaryKey = primaryKey;
             }
 
-            lastInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
+            lastInsertedPrimaryKey = primaryKey;
+
+            if(!CollectionUtils.nullSafeIsEmpty(insertedRecord.getWarnings()))
+            {
+               /////////////////////////////////////////////////////////////////////////////
+               // if there were warnings on the inserted record, put it in a warning line //
+               /////////////////////////////////////////////////////////////////////////////
+               String message = insertedRecord.getWarnings().get(0).getMessage();
+               processSummaryWarningsAndErrorsRollup.addWarning(message, primaryKey);
+            }
+            else
+            {
+               ////////////////////////////////////////////////////////////////////////
+               // if no warnings for the inserted record, then put it in the OK line //
+               ////////////////////////////////////////////////////////////////////////
+               okSummary.incrementCountAndAddPrimaryKey(primaryKey);
+            }
+         }
+         else
+         {
+            //////////////////////////////////////////////////////////////////////
+            // else if there were errors or no primary key, build an error line //
+            //////////////////////////////////////////////////////////////////////
+            String message = "Failed to insert";
+            if(!CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()))
+            {
+               //////////////////////////////////////////////////////////
+               // use the error message from the record if we have one //
+               //////////////////////////////////////////////////////////
+               message = insertedRecord.getErrors().get(0).getMessage();
+            }
+            processSummaryWarningsAndErrorsRollup.addError(message, primaryKey);
          }
       }
+
+      okSummary.pickMessage(true);
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java

@@ -24,8 +24,12 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
 
 import java.io.File;
 import java.io.InputStream;
+import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -37,9 +41,14 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapp
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadTableStructureBuilder;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import org.json.JSONObject;
 
 
 /*******************************************************************************
@@ -72,7 +81,7 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep
       {
          @SuppressWarnings("unchecked")
          List<String> headerValues = (List<String>) runBackendStepOutput.getValue("headerValues");
-         buildSuggestedMapping(headerValues, tableStructure, runBackendStepOutput);
+         buildSuggestedMapping(headerValues, getPrepopulatedValues(runBackendStepInput), tableStructure, runBackendStepOutput);
       }
    }
 
@@ -81,10 +90,62 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep
    /***************************************************************************
     **
     ***************************************************************************/
-   private void buildSuggestedMapping(List<String> headerValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput)
+   private Map<String, Serializable> getPrepopulatedValues(RunBackendStepInput runBackendStepInput)
+   {
+      String prepopulatedValuesJson = runBackendStepInput.getValueString("prepopulatedValues");
+      if(StringUtils.hasContent(prepopulatedValuesJson))
+      {
+         Map<String, Serializable> rs = new LinkedHashMap<>();
+         JSONObject jsonObject = JsonUtils.toJSONObject(prepopulatedValuesJson);
+         for(String key : jsonObject.keySet())
+         {
+            rs.put(key, jsonObject.optString(key, null));
+         }
+         return (rs);
+      }
+
+      return (Collections.emptyMap());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void buildSuggestedMapping(List<String> headerValues, Map<String, Serializable> prepopulatedValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput)
    {
       BulkLoadMappingSuggester bulkLoadMappingSuggester = new BulkLoadMappingSuggester();
       BulkLoadProfile          bulkLoadProfile          = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues);
+
+      if(CollectionUtils.nullSafeHasContents(prepopulatedValues))
+      {
+         for(Map.Entry<String, Serializable> entry : prepopulatedValues.entrySet())
+         {
+            String fieldName = entry.getKey();
+            boolean foundFieldInProfile = false;
+
+            for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList())
+            {
+               if(bulkLoadProfileField.getFieldName().equals(fieldName))
+               {
+                  foundFieldInProfile = true;
+                  bulkLoadProfileField.setColumnIndex(null);
+                  bulkLoadProfileField.setHeaderName(null);
+                  bulkLoadProfileField.setDefaultValue(entry.getValue());
+                  break;
+               }
+            }
+
+            if(!foundFieldInProfile)
+            {
+               BulkLoadProfileField bulkLoadProfileField = new BulkLoadProfileField();
+               bulkLoadProfileField.setFieldName(fieldName);
+               bulkLoadProfileField.setDefaultValue(entry.getValue());
+               bulkLoadProfile.getFieldList().add(bulkLoadProfileField);
+            }
+         }
+      }
+
       runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile);
       runBackendStepOutput.addValue("suggestedBulkLoadProfile", bulkLoadProfile);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java

@@ -52,6 +52,11 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep
    @Override
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
+      ////////////////////////////////////////////////////////////////////////////////////////
+      // for headless-bulk load (e.g., sftp import), set up the process tracer's key record //
+      ////////////////////////////////////////////////////////////////////////////////////////
+      runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput));
+
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // if user has come back here, clear out file (else the storageInput object that it is comes to the frontend, which isn't what we want!) //
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java

@@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mode
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.lang3.BooleanUtils;
 
@@ -77,10 +78,14 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
             //////////////////////////////////////////////////////////////////////////////
             if(savedBulkLoadProfileRecord == null)
             {
-               throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load"));
+               throw (new QUserFacingException("Did not receive a Bulk Load Profile record as input.  Unable to perform headless bulk load"));
             }
 
             SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord);
+            if(!StringUtils.hasContent(savedBulkLoadProfile.getMappingJson()))
+            {
+               throw (new QUserFacingException("Bulk Load Profile record's Mapping is empty.  Unable to perform headless bulk load"));
+            }
 
             try
             {
@@ -88,7 +93,7 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
             }
             catch(Exception e)
             {
-               throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e));
+               throw (new QUserFacingException("Error processing Bulk Load Profile record.  Unable to perform headless bulk load", e));
             }
          }
          else
@@ -240,6 +245,11 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
             }
          }
       }
+      catch(QUserFacingException ufe)
+      {
+         LOG.warn("User-facing error in bulk insert receive mapping", ufe);
+         throw ufe;
+      }
       catch(Exception e)
       {
          LOG.warn("Error in bulk insert receive mapping", e);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java

@@ -75,9 +75,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
  *******************************************************************************/
 public class BulkInsertTransformStep extends AbstractTransformStep
 {
-   private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
+   ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
 
-   private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted")
+   ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted")
       .withDoReplaceSingletonCountLinesWithSuffixOnly(false);
 
    private ListingHash<String, RowValue> errorToExampleRowValueMap = new ListingHash<>();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java

@@ -76,6 +76,23 @@ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows<org.dhatim.f
    }
 
 
+   /***************************************************************************
+    ** open/go-to a specific sheet (by 0-based index).  resets rows & iterator.
+    ***************************************************************************/
+   public void openSheet(int index) throws IOException
+   {
+      Optional<Sheet> sheet = workbook.getSheet(index);
+
+      if(sheet.isEmpty())
+      {
+         throw (new IOException("No sheet found for index: " + index));
+      }
+
+      rows = sheet.get().openStream();
+      setIterator(rows.iterator());
+   }
+
+
 
    /***************************************************************************
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java

@@ -53,7 +53,7 @@ public class BaseStreamedETLStep
    protected AbstractExtractStep getExtractStep(RunBackendStepInput runBackendStepInput)
    {
       QCodeReference codeReference = (QCodeReference) runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_EXTRACT_CODE);
-      return (QCodeLoader.getBackendStep(AbstractExtractStep.class, codeReference));
+      return (QCodeLoader.getAdHoc(AbstractExtractStep.class, codeReference));
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/CouldNotFindQueryFilterForExtractStepException.java

@@ -22,13 +22,13 @@
 package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
 
 
-import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class CouldNotFindQueryFilterForExtractStepException extends QException
+public class CouldNotFindQueryFilterForExtractStepException extends QUserFacingException
 {
    /*******************************************************************************
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java

@@ -279,7 +279,7 @@ public class ExtractViaQueryStep extends AbstractExtractStep
          return (new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, idStrings)));
       }
 
-      throw (new CouldNotFindQueryFilterForExtractStepException("Could not find query filter for Extract step."));
+      throw (new CouldNotFindQueryFilterForExtractStepException("No records were selected for running this process."));
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java

@@ -195,7 +195,7 @@ public class ProcessSummaryWarningsAndErrorsRollup
             {
                if(otherWarningsSummary == null)
                {
-                  otherWarningsSummary = new ProcessSummaryLine(Status.WARNING).withMessageSuffix("records had an other warning.");
+                  otherWarningsSummary = buildOtherWarningsSummary();
                }
                processSummaryLine = otherWarningsSummary;
             }
@@ -214,6 +214,27 @@ public class ProcessSummaryWarningsAndErrorsRollup
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static ProcessSummaryLine buildOtherWarningsSummary()
+   {
+      return new ProcessSummaryLine(Status.WARNING).withMessageSuffix("records had an other warning.");
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public void resetWarnings()
+   {
+      warningSummaries.clear();
+      otherWarningsSummary = buildOtherWarningsSummary();
+   }
+
+
+
    /*******************************************************************************
     ** Wrapper around AlphaNumericComparator for ProcessSummaryLineInterface that
     ** extracts string messages out.

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java

@@ -318,7 +318,7 @@ public class SavedReportToReportMetaDataAdapter
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static QReportField makeQReportField(String fieldName, FieldAndJoinTable fieldAndJoinTable)
+   public static QReportField makeQReportField(String fieldName, FieldAndJoinTable fieldAndJoinTable)
    {
       QReportField reportField = new QReportField();
 
@@ -404,5 +404,5 @@ public class SavedReportToReportMetaDataAdapter
    /*******************************************************************************
     **
     *******************************************************************************/
-   private record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable) {}
+   public record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable) {}
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java

@@ -356,12 +356,12 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
             {
                if(existingRecord != null)
                {
-                  LOG.info("Skipping storing existing record because this sync process is set to not perform updates");
+                  LOG.debug("Skipping storing existing record because this sync process is set to not perform updates");
                   willNotInsert.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
                }
                else
                {
-                  LOG.info("Skipping storing new record because this sync process is set to not perform inserts");
+                  LOG.debug("Skipping storing new record because this sync process is set to not perform inserts");
                   willNotUpdate.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
                }
                continue;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.utils;
 
 
 import java.io.Serializable;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -51,13 +52,14 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
  *******************************************************************************/
 public class RecordLookupHelper
 {
-   private Map<String, Map<Serializable, QRecord>> recordMaps = new HashMap<>();
+   private Map<String, Map<Serializable, QRecord>> recordMaps;
 
-   private Map<String, Map<Map<String, Serializable>, QRecord>> uniqueKeyMaps = new HashMap<>();
+   private Map<String, Map<Map<String, Serializable>, QRecord>> uniqueKeyMaps;
 
-   private Set<String> preloadedKeys = new HashSet<>();
+   private Set<String> preloadedKeys;
 
-   private Set<Pair<String, String>> disallowedOneOffLookups = new HashSet<>();
+   private Set<Pair<String, String>> disallowedOneOffLookups;
+   private boolean                   useSynchronizedCollections;
 
 
 
@@ -67,6 +69,33 @@ public class RecordLookupHelper
     *******************************************************************************/
    public RecordLookupHelper()
    {
+      this(false);
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public RecordLookupHelper(boolean useSynchronizedCollections)
+   {
+      this.useSynchronizedCollections = useSynchronizedCollections;
+
+      if(useSynchronizedCollections)
+      {
+         recordMaps = Collections.synchronizedMap(new HashMap<>());
+         uniqueKeyMaps = Collections.synchronizedMap(new HashMap<>());
+         preloadedKeys = Collections.synchronizedSet(new HashSet<>());
+         disallowedOneOffLookups = Collections.synchronizedSet(new HashSet<>());
+      }
+      else
+      {
+         recordMaps = new HashMap<>();
+         uniqueKeyMaps = new HashMap<>();
+         preloadedKeys = new HashSet<>();
+         disallowedOneOffLookups = new HashSet<>();
+      }
    }
 
 
@@ -77,7 +106,7 @@ public class RecordLookupHelper
    public QRecord getRecordByUniqueKey(String tableName, Map<String, Serializable> uniqueKey) throws QException
    {
       String                                  mapKey    = tableName + "." + uniqueKey.keySet().stream().sorted().collect(Collectors.joining(","));
-      Map<Map<String, Serializable>, QRecord> recordMap = uniqueKeyMaps.computeIfAbsent(mapKey, (k) -> new HashMap<>());
+      Map<Map<String, Serializable>, QRecord> recordMap = uniqueKeyMaps.computeIfAbsent(mapKey, (k) -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
 
       if(!recordMap.containsKey(uniqueKey))
       {
@@ -96,7 +125,7 @@ public class RecordLookupHelper
    public QRecord getRecordByKey(String tableName, String keyFieldName, Serializable key) throws QException
    {
       String                     mapKey    = tableName + "." + keyFieldName;
-      Map<Serializable, QRecord> recordMap = recordMaps.computeIfAbsent(mapKey, (k) -> new HashMap<>());
+      Map<Serializable, QRecord> recordMap = recordMaps.computeIfAbsent(mapKey, (k) -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
 
       ////////////////////////////////////////////////////////////
       // make sure we have they key object in the expected type //
@@ -150,7 +179,7 @@ public class RecordLookupHelper
    public void preloadRecords(String tableName, String keyFieldName, QQueryFilter filter) throws QException
    {
       String                     mapKey   = tableName + "." + keyFieldName;
-      Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>());
+      Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
       tableMap.putAll(GeneralProcessUtils.loadTableToMap(tableName, keyFieldName, filter));
    }
 
@@ -170,7 +199,7 @@ public class RecordLookupHelper
       }
 
       String                     mapKey   = tableName + "." + keyFieldName;
-      Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>());
+      Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
 
       QQueryFilter filter = new QQueryFilter(new QFilterCriteria(keyFieldName, QCriteriaOperator.IN, inList));
       tableMap.putAll(GeneralProcessUtils.loadTableToMap(tableName, keyFieldName, filter));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.state;
 import java.io.Serializable;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
@@ -58,7 +59,7 @@ public class InMemoryStateProvider implements StateProviderInterface
     *******************************************************************************/
    private InMemoryStateProvider()
    {
-      this.map = new HashMap<>();
+      this.map = Collections.synchronizedMap(new HashMap<>());
 
       ///////////////////////////////////////////////////////////
       // Start a single thread executor to handle the cleaning //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java

@@ -26,6 +26,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import com.ibm.icu.text.Transliterator;
 
 
 /*******************************************************************************
@@ -462,6 +463,17 @@ public class StringUtils
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static String replaceNonAsciiCharacters(String s)
+   {
+      Transliterator transliterator = Transliterator.getInstance("Any-Latin; Latin-ASCII");
+      return (transliterator.transliterate(s));
+   }
+
+
+
    /***************************************************************************
     **
     ***************************************************************************/
@@ -475,4 +487,49 @@ public class StringUtils
       return (s);
    }
 
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static boolean safeEqualsIgnoreCase(String a, String b)
+   {
+      if(a == null && b == null)
+      {
+         return true;
+      }
+      if(a == null || b == null)
+      {
+         return false;
+      }
+      return (a.equalsIgnoreCase(b));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static String appendIncrementingSuffix(String input)
+   {
+      ////////////////////////////////
+      // remove any existing suffix //
+      ////////////////////////////////
+      String base = input.replaceAll(" \\(\\d+\\)$", "");
+      if(input.matches(".* \\(\\d+\\)$"))
+      {
+         //////////////////////////
+         // increment if matches //
+         //////////////////////////
+         int current = Integer.parseInt(input.replaceAll(".* \\((\\d+)\\)$", "$1"));
+         return base + " (" + (current + 1) + ")";
+      }
+      else
+      {
+         ////////////////////////////////////
+         // no match so put a 1 at the end //
+         ////////////////////////////////////
+         return base + " (1)";
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/Timer.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.utils;
 
 
+import java.time.Duration;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import org.apache.logging.log4j.Level;
 
@@ -79,9 +80,36 @@ public class Timer
     **
     *******************************************************************************/
    public void mark(String message)
+   {
+      mark(message, false);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void mark(String message, boolean prettyPrint)
    {
       long now = System.currentTimeMillis();
-      LOG.log(level, String.format("%s: Last [%5d] Total [%5d] %s", name, (now - last), (now - start), message));
+
+      if(!prettyPrint)
+      {
+         LOG.log(level, String.format("%s: Last [%5d] Total [%5d] %s", name, (now - last), (now - start), message));
+      }
+      else
+      {
+
+         Duration lastDuration  = Duration.ofMillis(now - last);
+         Duration totalDuration = Duration.ofMillis(now - start);
+
+         LOG.log(level, String.format(
+            "%s: Last [%d hours, %d minutes, %d seconds, %d milliseconds] Total [%d hours, %d minutes, %d seconds, %d milliseconds] %s",
+            name, lastDuration.toHours(), lastDuration.toMinutesPart(), lastDuration.toSecondsPart(), lastDuration.toMillisPart(),
+            totalDuration.toHours(), totalDuration.toMinutesPart(), totalDuration.toSecondsPart(), totalDuration.toMillisPart(),
+            message));
+      }
+
       last = now;
    }
 }

qqq-backend-core/src/main/resources/log4j2.xml

@@ -18,21 +18,14 @@
       </File>
    </Appenders>
    <Loggers>
-      <Logger name="org.apache.log4j.xml" additivity="false">
-      </Logger>
-      <Logger name="org.mongodb.driver" level="WARN">
-      </Logger>
-      <Logger name="org.eclipse.jetty" level="INFO">
-      </Logger>
-      <Logger name="io.javalin" level="INFO">
-      </Logger>
+      <Logger name="org.mongodb.driver" level="WARN" />
+      <Logger name="org.eclipse.jetty" level="INFO" />
+      <Logger name="io.javalin" level="INFO" />
       <!-- c3p0 -->
-      <Logger name="com.mchange.v2" level="INFO">
-      </Logger>
-      <Logger name="org.quartz" level="INFO">
-      </Logger>
-      <Logger name="liquibase" level="INFO">
-      </Logger>
+      <Logger name="com.mchange.v2" level="INFO" />
+      <Logger name="org.quartz" level="INFO" />
+      <Logger name="liquibase" level="INFO" />
+      <Logger name="com.amazonaws" level="INFO" />
       <Root level="all">
          <AppenderRef ref="SystemOutAppender"/>
          <AppenderRef ref="SyslogAppender"/>

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoaderTest.java

@@ -22,12 +22,19 @@
 package com.kingsrook.qqq.backend.core.actions.customizers;
 
 
+import java.util.Map;
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.model.metadata.code.InitializableViaCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReferenceWithProperties;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.Timer;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
 
 /*******************************************************************************
@@ -80,6 +87,7 @@ class QCodeLoaderTest extends BaseTest
    }
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -91,4 +99,50 @@ class QCodeLoaderTest extends BaseTest
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testCodeReferenceWithProperties()
+   {
+      assertNull(QCodeLoader.getAdHoc(SomeClass.class, new QCodeReference(SomeClass.class)));
+
+      SomeClass someObject = QCodeLoader.getAdHoc(SomeClass.class, new QCodeReferenceWithProperties(SomeClass.class, Map.of("property", "someValue")));
+      assertEquals("someValue", someObject.someProperty);
+
+      SomeClass someOtherObject = QCodeLoader.getAdHoc(SomeClass.class, new QCodeReferenceWithProperties(SomeClass.class, Map.of("property", "someOtherValue")));
+      assertEquals("someOtherValue", someOtherObject.someProperty);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static class SomeClass implements InitializableViaCodeReference
+   {
+      private String someProperty;
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public void initialize(QCodeReference codeReference)
+      {
+         if(codeReference instanceof QCodeReferenceWithProperties codeReferenceWithProperties)
+         {
+            someProperty = ValueUtils.getValueAsString(codeReferenceWithProperties.getProperties().get("property"));
+         }
+
+         if(!StringUtils.hasContent(someProperty))
+         {
+            throw new IllegalStateException("Missing property");
+         }
+      }
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java

@@ -364,6 +364,7 @@ class MetaDataActionTest extends BaseTest
     **
     *******************************************************************************/
    @Test
+   @Deprecated(since = "migrated to metaDataCustomizer")
    void testFilter() throws QException
    {
       QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(MetaDataAction.class);
@@ -397,7 +398,7 @@ class MetaDataActionTest extends BaseTest
       // run again (with the same instance as before) to assert about memoization of the filter based on the QInstance //
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       new MetaDataAction().execute(new MetaDataInput());
-      assertThat(collectingLogger.getCollectedMessages()).filteredOn(clm -> clm.getMessage().contains("filter of type: DenyAllFilter")).hasSize(1);
+      assertThat(collectingLogger.getCollectedMessages()).filteredOn(clm -> clm.getMessage().contains("actionCustomizer (via metaDataFilter reference) of type: DenyAllFilter")).hasSize(1);
 
       QLogger.deactivateCollectingLoggerForClass(MetaDataAction.class);
 
@@ -413,6 +414,59 @@ class MetaDataActionTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testCustomizer() throws QException
+   {
+      QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(MetaDataAction.class);
+
+      //////////////////////////////////////////////////////
+      // run default version, and assert tables are found //
+      //////////////////////////////////////////////////////
+      MetaDataOutput result = new MetaDataAction().execute(new MetaDataInput());
+      assertFalse(result.getTables().isEmpty(), "should be some tables");
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // run again (with the same instance as before) to assert about memoization of the filter based on the QInstance //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      new MetaDataAction().execute(new MetaDataInput());
+      assertThat(collectingLogger.getCollectedMessages()).filteredOn(clm -> clm.getMessage().contains("Using new default")).hasSize(1);
+
+      /////////////////////////////////////////////////////////////
+      // set up new instance to use a custom filter, to deny all //
+      /////////////////////////////////////////////////////////////
+      QInstance instance = TestUtils.defineInstance();
+      instance.setMetaDataActionCustomizer(new QCodeReference(DenyAllFilteringCustomizer.class));
+      reInitInstanceInContext(instance);
+
+      /////////////////////////////////////////////////////
+      // re-run, and assert all tables are filtered away //
+      /////////////////////////////////////////////////////
+      result = new MetaDataAction().execute(new MetaDataInput());
+      assertTrue(result.getTables().isEmpty(), "should be no tables");
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // run again (with the same instance as before) to assert about memoization of the filter based on the QInstance //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      new MetaDataAction().execute(new MetaDataInput());
+      assertThat(collectingLogger.getCollectedMessages()).filteredOn(clm -> clm.getMessage().contains("meta-data actionCustomizer of type: DenyAllFilteringCustomizer")).hasSize(1);
+
+      QLogger.deactivateCollectingLoggerForClass(MetaDataAction.class);
+
+      /////////////////////////////////////////////////////////////////////////////////
+      // run now with the DefaultNoopMetaDataActionCustomizer, confirm we get tables //
+      /////////////////////////////////////////////////////////////////////////////////
+      instance = TestUtils.defineInstance();
+      instance.setMetaDataActionCustomizer(new QCodeReference(DefaultNoopMetaDataActionCustomizer.class));
+      reInitInstanceInContext(instance);
+      result = new MetaDataAction().execute(new MetaDataInput());
+      assertFalse(result.getTables().isEmpty(), "should be some tables");
+   }
+
+
+
    /***************************************************************************
     **
     ***************************************************************************/
@@ -462,6 +516,67 @@ class MetaDataActionTest extends BaseTest
 
 
 
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public boolean allowWidget(MetaDataInput input, QWidgetMetaDataInterface widget)
+      {
+         return false;
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static class DenyAllFilteringCustomizer implements MetaDataActionCustomizerInterface
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public boolean allowTable(MetaDataInput input, QTableMetaData table)
+      {
+         return false;
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public boolean allowProcess(MetaDataInput input, QProcessMetaData process)
+      {
+         return false;
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public boolean allowReport(MetaDataInput input, QReportMetaData report)
+      {
+         return false;
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public boolean allowApp(MetaDataInput input, QAppMetaData app)
+      {
+         return false;
+      }
+
+
+
       /***************************************************************************
        **
        ***************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java

@@ -38,6 +38,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.reporting.excel.TestExcelStyler;
 import com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel.ExcelFastexcelExportStreamer;
 import com.kingsrook.qqq.backend.core.actions.reporting.excel.poi.BoldHeaderAndFooterPoiExcelStyler;
 import com.kingsrook.qqq.backend.core.actions.reporting.excel.poi.ExcelPoiBasedStreamingExportStreamer;
@@ -56,6 +57,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
@@ -490,6 +492,34 @@ public class GenerateReportActionTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void runXlsxWithStyleCustomizer() throws Exception
+   {
+      ReportFormat format = ReportFormat.XLSX;
+      String       name   = "/tmp/report-customized.xlsx";
+      try(FileOutputStream fileOutputStream = new FileOutputStream(name))
+      {
+         QInstance qInstance = QContext.getQInstance();
+         qInstance.addReport(defineTableOnlyReport());
+         insertPersonRecords(qInstance);
+
+         ReportInput reportInput = new ReportInput();
+         reportInput.setReportName(REPORT_NAME);
+         reportInput.setReportDestination(new ReportDestination().withReportFormat(format).withReportOutputStream(fileOutputStream));
+         reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now()));
+         reportInput.setExportStyleCustomizer(new QCodeReference(TestExcelStyler.class));
+         new GenerateReportAction().execute(reportInput);
+         System.out.println("Wrote File: " + name);
+
+         LocalMacDevUtils.openFile(name);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/TestExcelStyler.java

@@ -0,0 +1,76 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excel;
+
+
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.reporting.excel.poi.ExcelPoiBasedStreamingStyleCustomizerInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
+import org.apache.poi.ss.usermodel.CreationHelper;
+import org.apache.poi.ss.usermodel.Font;
+import org.apache.poi.xssf.usermodel.XSSFCellStyle;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class TestExcelStyler implements ExcelPoiBasedStreamingStyleCustomizerInterface
+{
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public List<Integer> getColumnWidthsForView(QReportView view)
+   {
+      return List.of(60, 50, 40);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public List<String> getMergedRangesForView(QReportView view)
+   {
+      return List.of("A1:B1");
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void customizeStyles(Map<String, XSSFCellStyle> styles, XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      Font font = workbook.createFont();
+      font.setFontHeightInPoints((short) 16);
+      font.setBold(true);
+      XSSFCellStyle cellStyle = workbook.createCellStyle();
+      cellStyle.setFont(font);
+      styles.put("header", cellStyle);
+   }
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java

@@ -79,6 +79,7 @@ class ConvertHtmlToPdfActionTest extends BaseTest
                      </h1>
                      <div class="myclass">
                          <p>This is a test of converting HTML to PDF!!</p>
+                         <p>This is &nbsp; a line with &bull; some entities &lt;</p>
                          <p style="font-family: SF-Pro; font-size: 24px;">(btw, is this in SF-Pro???)</p>
                      </div>
                  </div>

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -40,6 +40,7 @@ import com.kingsrook.qqq.backend.core.actions.dashboard.PersonsByCreateDateBarCh
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ParentWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.metadata.AllowAllMetaDataFilter;
+import com.kingsrook.qqq.backend.core.actions.metadata.DefaultNoopMetaDataActionCustomizer;
 import com.kingsrook.qqq.backend.core.actions.processes.CancelProcessActionTest;
 import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
@@ -160,6 +161,20 @@ public class QInstanceValidatorTest extends BaseTest
    }
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testMetaDataActionCustomizer()
+   {
+      assertValidationFailureReasons((qInstance) -> qInstance.setMetaDataActionCustomizer(new QCodeReference(QInstanceValidator.class)),
+         "Instance metaDataActionCustomizer CodeReference is not of the expected type");
+
+      assertValidationSuccess((qInstance) -> qInstance.setMetaDataActionCustomizer(new QCodeReference(DefaultNoopMetaDataActionCustomizer.class)));
+      assertValidationSuccess((qInstance) -> qInstance.setMetaDataActionCustomizer(null));
+   }
+
+
 
    /*******************************************************************************
     ** Test an instance with null backends - should throw.

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java

@@ -185,4 +185,13 @@ public class ProcessSummaryLineInterfaceAssert extends AbstractAssert<ProcessSum
       return (this);
    }
 
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public ProcessSummaryLineInterface getLine()
+   {
+      return actual;
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java

@@ -41,6 +41,7 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -566,4 +567,22 @@ class QRecordEntityTest extends BaseTest
       assertEquals(0, order.getLineItems().size());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTableName() throws QException
+   {
+      assertEquals(Item.TABLE_NAME, QRecordEntity.getTableName(Item.class));
+      assertEquals(Item.TABLE_NAME, Item.getTableName(Item.class));
+      assertEquals(Item.TABLE_NAME, new Item().tableName());
+
+      //////////////////////////////////
+      // no TABLE_NAME in Order class //
+      //////////////////////////////////
+      assertThatThrownBy(() -> Order.getTableName(Order.class));
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/WhiteSpaceBehaviorTest.java

@@ -0,0 +1,280 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.fields;
+
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for WhiteSpaceBehavior
+ *******************************************************************************/
+class WhiteSpaceBehaviorTest extends BaseTest
+{
+   public static final String FIELD = "firstName";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testNone()
+   {
+      assertNull(applyToRecord(WhiteSpaceBehavior.NONE, new QRecord(), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertNull(applyToRecord(WhiteSpaceBehavior.NONE, new QRecord().withValue(FIELD, null), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertEquals("John", applyToRecord(WhiteSpaceBehavior.NONE, new QRecord().withValue(FIELD, "John"), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+
+      assertEquals(ListBuilder.of("J.   ohn", null, "Jane\n"), applyToRecords(WhiteSpaceBehavior.NONE, List.of(
+            new QRecord().withValue(FIELD, "J.   ohn"),
+            new QRecord(),
+            new QRecord().withValue(FIELD, "Jane\n")),
+         ValueBehaviorApplier.Action.INSERT).stream().map(r -> r.getValueString(FIELD)).toList());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testRemoveWhiteSpace()
+   {
+      assertNull(applyToRecord(WhiteSpaceBehavior.REMOVE_ALL_WHITESPACE, new QRecord(), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertNull(applyToRecord(WhiteSpaceBehavior.REMOVE_ALL_WHITESPACE, new QRecord().withValue(FIELD, null), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertEquals("doobeedoobeedoo", applyToRecord(WhiteSpaceBehavior.REMOVE_ALL_WHITESPACE, new QRecord().withValue(FIELD, "doo bee doo\n bee doo"), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+
+      assertEquals(ListBuilder.of("thisistheway", null, "thatwastheway"), applyToRecords(WhiteSpaceBehavior.REMOVE_ALL_WHITESPACE, List.of(
+            new QRecord().withValue(FIELD, "this is\rthe way   \t"),
+            new QRecord(),
+            new QRecord().withValue(FIELD, "that was the way\n")),
+         ValueBehaviorApplier.Action.INSERT).stream().map(r -> r.getValueString(FIELD)).toList());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTrimWhiteSpace()
+   {
+      assertNull(applyToRecord(WhiteSpaceBehavior.TRIM, new QRecord(), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertNull(applyToRecord(WhiteSpaceBehavior.TRIM, new QRecord().withValue(FIELD, null), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertEquals("doo bee doo\n bee doo", applyToRecord(WhiteSpaceBehavior.TRIM, new QRecord().withValue(FIELD, "    doo bee doo\n bee doo\r \n\n"), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+
+      assertEquals(ListBuilder.of("this is\rthe way", null, "that was the way"), applyToRecords(WhiteSpaceBehavior.TRIM, List.of(
+            new QRecord().withValue(FIELD, "      this is\rthe way   \t"),
+            new QRecord(),
+            new QRecord().withValue(FIELD, "that was the way\n")),
+         ValueBehaviorApplier.Action.INSERT).stream().map(r -> r.getValueString(FIELD)).toList());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTrimLeftWhiteSpace()
+   {
+      assertNull(applyToRecord(WhiteSpaceBehavior.TRIM_LEFT, new QRecord(), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertNull(applyToRecord(WhiteSpaceBehavior.TRIM_LEFT, new QRecord().withValue(FIELD, null), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertEquals("doo bee doo\n bee doo\r \n\n", applyToRecord(WhiteSpaceBehavior.TRIM_LEFT, new QRecord().withValue(FIELD, "    doo bee doo\n bee doo\r \n\n"), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+
+      assertEquals(ListBuilder.of("this is\rthe way   \t", null, "that was the way\n"), applyToRecords(WhiteSpaceBehavior.TRIM_LEFT, List.of(
+            new QRecord().withValue(FIELD, "      this is\rthe way   \t"),
+            new QRecord(),
+            new QRecord().withValue(FIELD, "  \n that was the way\n")),
+         ValueBehaviorApplier.Action.INSERT).stream().map(r -> r.getValueString(FIELD)).toList());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTrimRightWhiteSpace()
+   {
+      assertNull(applyToRecord(WhiteSpaceBehavior.TRIM_RIGHT, new QRecord(), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertNull(applyToRecord(WhiteSpaceBehavior.TRIM_RIGHT, new QRecord().withValue(FIELD, null), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertEquals("    doo bee doo\n bee doo", applyToRecord(WhiteSpaceBehavior.TRIM_RIGHT, new QRecord().withValue(FIELD, "    doo bee doo\n bee doo\r \n\n"), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+
+      assertEquals(ListBuilder.of("      this is\rthe way", null, "  \n that was the way"), applyToRecords(WhiteSpaceBehavior.TRIM_RIGHT, List.of(
+            new QRecord().withValue(FIELD, "      this is\rthe way   \t"),
+            new QRecord(),
+            new QRecord().withValue(FIELD, "  \n that was the way\n")),
+         ValueBehaviorApplier.Action.INSERT).stream().map(r -> r.getValueString(FIELD)).toList());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QRecord applyToRecord(WhiteSpaceBehavior behavior, QRecord record, ValueBehaviorApplier.Action action)
+   {
+      return (applyToRecords(behavior, List.of(record), action).get(0));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<QRecord> applyToRecords(WhiteSpaceBehavior behavior, List<QRecord> records, ValueBehaviorApplier.Action action)
+   {
+      QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      behavior.apply(action, records, QContext.getQInstance(), table, table.getField(FIELD));
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testReads() throws QException
+   {
+      TestUtils.insertDefaultShapes(QContext.getQInstance());
+
+      List<QRecord> records = QueryAction.execute(TestUtils.TABLE_NAME_SHAPE, null);
+      assertEquals(Set.of("Triangle", "Square", "Circle"), records.stream().map(r -> r.getValueString("name")).collect(Collectors.toSet()));
+
+      QFieldMetaData field = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_SHAPE).getField("name");
+      field.setBehaviors(Set.of(CaseChangeBehavior.TO_UPPER_CASE));
+
+      records = QueryAction.execute(TestUtils.TABLE_NAME_SHAPE, null);
+      assertEquals(Set.of("TRIANGLE", "SQUARE", "CIRCLE"), records.stream().map(r -> r.getValueString("name")).collect(Collectors.toSet()));
+
+      field.setBehaviors(Set.of(CaseChangeBehavior.TO_LOWER_CASE));
+      assertEquals("triangle", GetAction.execute(TestUtils.TABLE_NAME_SHAPE, 1).getValueString("name"));
+
+      field.setBehaviors(Set.of(CaseChangeBehavior.NONE));
+      assertEquals("Triangle", GetAction.execute(TestUtils.TABLE_NAME_SHAPE, 1).getValueString("name"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testWrites() throws QException
+   {
+      Integer id = 100;
+
+      QFieldMetaData field = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_SHAPE).getField("name");
+      field.setBehaviors(Set.of(CaseChangeBehavior.TO_UPPER_CASE));
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_SHAPE).withRecord(new QRecord().withValue("id", id).withValue("name", "Octagon")));
+
+      //////////////////////////////////////////////////////////////////////////////////
+      // turn off the to-upper-case behavior, so we'll see what was actually inserted //
+      //////////////////////////////////////////////////////////////////////////////////
+      field.setBehaviors(Collections.emptySet());
+      assertEquals("OCTAGON", GetAction.execute(TestUtils.TABLE_NAME_SHAPE, id).getValueString("name"));
+
+      ////////////////////////////////////////////
+      // change to toLowerCase and do an update //
+      ////////////////////////////////////////////
+      field.setBehaviors(Set.of(CaseChangeBehavior.TO_LOWER_CASE));
+      new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_SHAPE).withRecord(new QRecord().withValue("id", id).withValue("name", "Octagon")));
+
+      ////////////////////////////////////////////////////////////////////////////////////
+      // turn off the to-lower-case behavior, so we'll see what was actually updated to //
+      ////////////////////////////////////////////////////////////////////////////////////
+      field.setBehaviors(Collections.emptySet());
+      assertEquals("octagon", GetAction.execute(TestUtils.TABLE_NAME_SHAPE, id).getValueString("name"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFilter()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_SHAPE);
+      QFieldMetaData field     = table.getField("name");
+      field.setBehaviors(Set.of(CaseChangeBehavior.TO_UPPER_CASE));
+      assertEquals("SQUARE", CaseChangeBehavior.TO_UPPER_CASE.applyToFilterCriteriaValue("square", qInstance, table, field));
+
+      field.setBehaviors(Set.of(CaseChangeBehavior.TO_LOWER_CASE));
+      assertEquals("triangle", CaseChangeBehavior.TO_LOWER_CASE.applyToFilterCriteriaValue("Triangle", qInstance, table, field));
+
+      field.setBehaviors(Set.of(CaseChangeBehavior.NONE));
+      assertEquals("Circle", CaseChangeBehavior.NONE.applyToFilterCriteriaValue("Circle", qInstance, table, field));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testValidation()
+   {
+      QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_SHAPE);
+
+      ///////////////////////////////////////////
+      // should be no errors on a string field //
+      ///////////////////////////////////////////
+      assertTrue(CaseChangeBehavior.TO_UPPER_CASE.validateBehaviorConfiguration(table, table.getField("name")).isEmpty());
+
+      //////////////////////////////////////////
+      // should be an error on a number field //
+      //////////////////////////////////////////
+      assertEquals(1, CaseChangeBehavior.TO_LOWER_CASE.validateBehaviorConfiguration(table, table.getField("id")).size());
+
+      /////////////////////////////////////////
+      // NONE should be allowed on any field //
+      /////////////////////////////////////////
+      assertTrue(CaseChangeBehavior.NONE.validateBehaviorConfiguration(table, table.getField("id")).isEmpty());
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java

@@ -29,15 +29,26 @@ import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
 import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
@@ -176,7 +187,15 @@ class BulkInsertFullProcessTest extends BaseTest
       assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class);
       assertThat(runProcessOutput.getException()).isEmpty();
 
-      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id values between 1 and 2");
+      ProcessSummaryLineInterface okLine = ProcessSummaryAssert.assertThat(runProcessOutput)
+         .hasLineWithMessageContaining("Person Memory records were inserted")
+         .hasStatus(Status.OK)
+         .hasCount(2)
+         .getLine();
+      assertEquals(List.of(1, 2), ((ProcessSummaryLine) okLine).getPrimaryKeys());
+
+      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("records were processed from the file").hasStatus(Status.INFO);
+      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id values between 1 and 2").hasStatus(Status.INFO);
 
       ////////////////////////////////////
       // query for the inserted records //
@@ -201,6 +220,86 @@ class BulkInsertFullProcessTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSummaryLinePrimaryKeys() throws Exception
+   {
+      assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty();
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
+         .withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(PersonWarnOrErrorCustomizer.class));
+
+      /////////////////////////////////////////////////////////
+      // start the process - expect to go to the upload step //
+      /////////////////////////////////////////////////////////
+      RunProcessInput  runProcessInput  = new RunProcessInput();
+      RunProcessOutput runProcessOutput = startProcess(runProcessInput);
+      String           processUUID      = runProcessOutput.getProcessUUID();
+
+      continueProcessPostUpload(runProcessInput, processUUID, simulateFileUploadForWarningCase());
+      continueProcessPostFileMapping(runProcessInput);
+      continueProcessPostValueMapping(runProcessInput);
+      runProcessOutput = continueProcessPostReviewScreen(runProcessInput);
+
+      ProcessSummaryLineInterface okLine = ProcessSummaryAssert.assertThat(runProcessOutput)
+         .hasLineWithMessageContaining("Person Memory record was inserted")
+         .hasStatus(Status.OK)
+         .hasCount(1)
+         .getLine();
+      assertEquals(List.of(1), ((ProcessSummaryLine) okLine).getPrimaryKeys());
+
+      ProcessSummaryLineInterface warnTornadoLine = ProcessSummaryAssert.assertThat(runProcessOutput)
+         .hasLineWithMessageContaining("records were inserted, but had a warning: Tornado warning")
+         .hasStatus(Status.WARNING)
+         .hasCount(2)
+         .getLine();
+      assertEquals(List.of(2, 3), ((ProcessSummaryLine) warnTornadoLine).getPrimaryKeys());
+
+      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("record was inserted, but had a warning: Hurricane warning").hasStatus(Status.WARNING).hasCount(1);
+      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("records were processed from the file").hasStatus(Status.INFO).hasCount(4);
+      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id values between 1 and 4").hasStatus(Status.INFO);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSummaryLineErrors() throws Exception
+   {
+      assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty();
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
+         .withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(PersonWarnOrErrorCustomizer.class));
+
+      /////////////////////////////////////////////////////////
+      // start the process - expect to go to the upload step //
+      /////////////////////////////////////////////////////////
+      RunProcessInput  runProcessInput  = new RunProcessInput();
+      RunProcessOutput runProcessOutput = startProcess(runProcessInput);
+      String           processUUID      = runProcessOutput.getProcessUUID();
+
+      continueProcessPostUpload(runProcessInput, processUUID, simulateFileUploadForErrorCase());
+      continueProcessPostFileMapping(runProcessInput);
+      continueProcessPostValueMapping(runProcessInput);
+      runProcessOutput = continueProcessPostReviewScreen(runProcessInput);
+
+      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Person Memory record was inserted.").hasStatus(Status.OK).hasCount(1);
+
+      ProcessSummaryAssert.assertThat(runProcessOutput)
+         .hasLineWithMessageContaining("plane")
+         .hasStatus(Status.ERROR)
+         .hasCount(1);
+
+      ProcessSummaryAssert.assertThat(runProcessOutput)
+         .hasLineWithMessageContaining("purifier")
+         .hasStatus(Status.ERROR)
+         .hasCount(1);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -301,6 +400,47 @@ class BulkInsertFullProcessTest extends BaseTest
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static StorageInput simulateFileUploadForWarningCase() throws Exception
+   {
+      String       storageReference = UUID.randomUUID() + ".csv";
+      StorageInput storageInput     = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference);
+      try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput))
+      {
+         outputStream.write((getPersonCsvHeaderUsingLabels() + """
+            "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com","Missouri",42
+            "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Tornado warning","Doe","1980-01-01","john@doe.com","Missouri",42
+            "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Tornado warning","Doey","1980-01-01","john@doe.com","Missouri",42
+            "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Hurricane warning","Doe","1980-01-01","john@doe.com","Missouri",42
+            """).getBytes());
+      }
+      return storageInput;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static StorageInput simulateFileUploadForErrorCase() throws Exception
+   {
+      String       storageReference = UUID.randomUUID() + ".csv";
+      StorageInput storageInput     = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference);
+      try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput))
+      {
+         outputStream.write((getPersonCsvHeaderUsingLabels() + """
+            "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com","Missouri",42
+            "0","2021-10-26 14:39:37","2021-10-26 14:39:37","not-pre-Error plane","Doe","1980-01-01","john@doe.com","Missouri",42
+            "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Error purifier","Doe","1980-01-01","john@doe.com","Missouri",42
+            """).getBytes());
+      }
+      return storageInput;
+   }
+
+
+
    /***************************************************************************
     **
     ***************************************************************************/
@@ -331,4 +471,47 @@ class BulkInsertFullProcessTest extends BaseTest
       )));
    }
 
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static class PersonWarnOrErrorCustomizer implements TableCustomizerInterface
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public AbstractPreInsertCustomizer.WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview)
+      {
+         return AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS;
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
+      {
+         for(QRecord record : records)
+         {
+            if(record.getValueString("firstName").toLowerCase().contains("warn"))
+            {
+               record.addWarning(new QWarningMessage(record.getValueString("firstName")));
+            }
+            else if(record.getValueString("firstName").toLowerCase().contains("error"))
+            {
+               if(isPreview && record.getValueString("firstName").toLowerCase().contains("not-pre-error"))
+               {
+                  continue;
+               }
+
+               record.addError(new BadInputStatusMessage(record.getValueString("firstName")));
+            }
+         }
+         return records;
+      }
+   }
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java

@@ -22,8 +22,22 @@
 package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
 
 
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
@@ -66,4 +80,39 @@ class BulkInsertPrepareFileMappingStepTest extends BaseTest
       assertEquals("BAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 * 26 + 26 + 0));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws Exception
+   {
+      String fileName = "personFile.csv";
+
+      StorageInput    storageInput    = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(fileName);
+      OutputStream    outputStream    = new StorageAction().createOutputStream(storageInput);
+      outputStream.write("""
+         name,noOfShoes
+         John,2
+         Jane,4
+         """.getBytes(StandardCharsets.UTF_8));
+      outputStream.close();
+
+      RunProcessInput runProcessInput = new RunProcessInput();
+      BulkInsertStepUtils.setStorageInputForTheFile(runProcessInput, storageInput);
+      runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      runProcessInput.addValue("prepopulatedValues", JsonUtils.toJson(Map.of("homeStateId", 1)));
+
+      RunBackendStepInput runBackendStepInput = new RunBackendStepInput(runProcessInput.getProcessState());
+      RunBackendStepOutput runBackendStepOutput = new RunBackendStepOutput();
+
+      new BulkInsertPrepareFileMappingStep().run(runBackendStepInput, runBackendStepOutput);
+
+      BulkLoadProfile bulkLoadProfile = (BulkLoadProfile) runBackendStepOutput.getValue("suggestedBulkLoadProfile");
+      Optional<BulkLoadProfileField> homeStateId = bulkLoadProfile.getFieldList().stream().filter(f -> f.getFieldName().equals("homeStateId")).findFirst();
+      assertThat(homeStateId).isPresent();
+      assertEquals("1", homeStateId.get().getDefaultValue());
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java

@@ -22,8 +22,15 @@
 package com.kingsrook.qqq.backend.core.processes.utils;
 
 
+import java.util.ArrayList;
+import java.util.ConcurrentModificationException;
 import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -32,6 +39,7 @@ import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Fail.fail;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
@@ -195,4 +203,54 @@ class RecordLookupHelperTest extends BaseTest
       assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
    }
 
+
+
+   /*******************************************************************************
+    ** run a lot of threads (eg, 100), each trying to do lots of work in a
+    ** shared recordLookupHelper.  w/o the flag to use sync'ed collections, this
+    ** (usually?) fails with a ConcurrentModificationException - but with the sync'ed
+    ** collections, is safe.
+    *******************************************************************************/
+   @Test
+   void testConcurrentModification() throws InterruptedException, ExecutionException
+   {
+      ExecutorService    executorService    = Executors.newFixedThreadPool(100);
+      RecordLookupHelper recordLookupHelper = new RecordLookupHelper(true);
+
+      CapturedContext capture = QContext.capture();
+
+      List<Future<?>> futures = new ArrayList<>();
+      for(int i = 0; i < 100; i++)
+      {
+         int finalI = i;
+         Future<?> future = executorService.submit(() ->
+         {
+            QContext.init(capture);
+            for(int j = 0; j < 25000; j++)
+            {
+               try
+               {
+                  recordLookupHelper.getRecordByKey(String.valueOf(j), "id", j);
+               }
+               catch(ConcurrentModificationException cme)
+               {
+                  fail("CME!", cme);
+               }
+               catch(Exception e)
+               {
+                  //////////////
+                  // expected //
+                  //////////////
+               }
+            }
+         });
+         futures.add(future);
+      }
+
+      for(Future<?> future : futures)
+      {
+         future.get();
+      }
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java

@@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.state;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -103,17 +105,17 @@ public class InMemoryStateProviderTest extends BaseTest
       /////////////////////////////////////////////////////////////
       // Add an entry that is 3 hours old, should not be cleaned //
       /////////////////////////////////////////////////////////////
-      UUIDAndTypeStateKey   newKey           = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(3, ChronoUnit.HOURS));
-      String  newUUID    = UUID.randomUUID().toString();
-      QRecord newQRecord = new QRecord().withValue("uuid", newUUID);
+      UUIDAndTypeStateKey newKey     = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(3, ChronoUnit.HOURS));
+      String              newUUID    = UUID.randomUUID().toString();
+      QRecord             newQRecord = new QRecord().withValue("uuid", newUUID);
       stateProvider.put(newKey, newQRecord);
 
       ////////////////////////////////////////////////////////////
       // Add an entry that is 5 hours old, it should be cleaned //
       ////////////////////////////////////////////////////////////
-      UUIDAndTypeStateKey   oldKey           = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(5, ChronoUnit.HOURS));
-      String  oldUUID    = UUID.randomUUID().toString();
-      QRecord oldQRecord = new QRecord().withValue("uuid", oldUUID);
+      UUIDAndTypeStateKey oldKey     = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(5, ChronoUnit.HOURS));
+      String              oldUUID    = UUID.randomUUID().toString();
+      QRecord             oldQRecord = new QRecord().withValue("uuid", oldUUID);
       stateProvider.put(oldKey, oldQRecord);
 
       ///////////////////
@@ -125,7 +127,33 @@ public class InMemoryStateProviderTest extends BaseTest
       Assertions.assertEquals(newUUID, qRecordFromState.getValueString("uuid"), "Should read value from state persistence");
 
       Assertions.assertTrue(stateProvider.get(QRecord.class, oldKey).isEmpty(), "Key not found in state should return empty");
+   }
 
+
+
+   /*******************************************************************************
+    ** originally written with N=100000, but showed the error as small as 1000.
+    *******************************************************************************/
+   @Test
+   void testDemonstrateConcurrentModificationIfNonSynchronizedMap()
+   {
+      int N = 1000;
+      InMemoryStateProvider stateProvider = InMemoryStateProvider.getInstance();
+
+      ExecutorService executorService = Executors.newFixedThreadPool(10);
+      executorService.submit(() ->
+      {
+         for(int i = 0; i < N; i++)
+         {
+            UUIDAndTypeStateKey oldKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(5, ChronoUnit.HOURS));
+            stateProvider.put(oldKey, UUID.randomUUID());
+         }
+      });
+
+      for(int i = 0; i < N; i++)
+      {
+         stateProvider.clean(Instant.now().minus(4, ChronoUnit.HOURS));
+      }
    }
 
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java

@@ -334,4 +334,38 @@ class StringUtilsTest extends BaseTest
       assertEquals("a", StringUtils.emptyToNull("a"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testAppendIncrementingSuffix()
+   {
+      assertEquals("test (1)", StringUtils.appendIncrementingSuffix("test"));
+      assertEquals("test (2)", StringUtils.appendIncrementingSuffix("test (1)"));
+      assertEquals("test (a) (1)", StringUtils.appendIncrementingSuffix("test (a)"));
+      assertEquals("test (a32) (1)", StringUtils.appendIncrementingSuffix("test (a32)"));
+      assertEquals("test ((2)) (1)", StringUtils.appendIncrementingSuffix("test ((2))"));
+      assertEquals("test ((2)) (101)", StringUtils.appendIncrementingSuffix("test ((2)) (100)"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSafeEqualsIgnoreCase()
+   {
+      assertTrue(StringUtils.safeEqualsIgnoreCase(null, null));
+      assertFalse(StringUtils.safeEqualsIgnoreCase("a", null));
+      assertFalse(StringUtils.safeEqualsIgnoreCase(null, "a"));
+      assertTrue(StringUtils.safeEqualsIgnoreCase("a", "a"));
+      assertTrue(StringUtils.safeEqualsIgnoreCase("A", "a"));
+      assertFalse(StringUtils.safeEqualsIgnoreCase("a", "b"));
+      assertTrue(StringUtils.safeEqualsIgnoreCase("timothy d. chamberlain", "TIMOThy d. chaMberlain"));
+      assertTrue(StringUtils.safeEqualsIgnoreCase("timothy d. chamberlain", "timothy d. chamberlain"));
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java

@@ -362,4 +362,4 @@ class ValueUtilsTest extends BaseTest
       assertEquals(QFieldType.TIME, ValueUtils.inferQFieldTypeFromValue(LocalTime.now(), null));
    }
 
-}
+}

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java

@@ -39,6 +39,7 @@ import java.util.function.Supplier;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
@@ -208,6 +209,11 @@ public class BaseAPIActionUtil
 
          return rs;
       }
+      catch(QNotFoundException qnfe)
+      {
+         LOG.info("Not found", qnfe);
+         throw (qnfe);
+      }
       catch(Exception e)
       {
          LOG.error("Error in API get", e);
@@ -737,6 +743,10 @@ public class BaseAPIActionUtil
          case OAUTH2 -> request.setHeader("Authorization", "Bearer " + getOAuth2Token());
          case API_KEY_QUERY_PARAM -> addApiKeyQueryParamToRequest(request);
          case CUSTOM -> handleCustomAuthorization(request);
+         case NONE ->
+         {
+            /* nothing to do here */
+         }
          default -> throw new IllegalArgumentException("Unexpected authorization type: " + backendMetaData.getAuthorizationType());
       }
    }
@@ -1168,6 +1178,16 @@ public class BaseAPIActionUtil
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected QHttpResponse getQHttpResponse(HttpResponse response) throws Exception
+   {
+      return (new QHttpResponse(response));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -1201,7 +1221,7 @@ public class BaseAPIActionUtil
 
             try(CloseableHttpResponse response = executeHttpRequest(request, httpClient))
             {
-               QHttpResponse qResponse = new QHttpResponse(response);
+               QHttpResponse qResponse = getQHttpResponse(response);
 
                logOutboundApiCall(request, qResponse);
 

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/QHttpResponse.java

@@ -40,6 +40,7 @@ public class QHttpResponse
    private String       statusReasonPhrase;
    private List<Header> headerList;
    private String       content;
+   private byte[]       contentBytes;
 
 
 
@@ -53,11 +54,47 @@ public class QHttpResponse
 
 
 
+   /*******************************************************************************
+    ** Constructor for QHttpResponse that allows reading content as bytes
+    **
+    *******************************************************************************/
+   public QHttpResponse(HttpResponse httpResponse, boolean readContentAsBytes) throws Exception
+   {
+      if(!readContentAsBytes)
+      {
+         new QHttpResponse(httpResponse);
+         return;
+      }
+
+      setGeneralHttpResponseData(httpResponse);
+      if(this.statusCode == null || this.statusCode != 204)
+      {
+         this.contentBytes = httpResponse.getEntity().getContent().readAllBytes();
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Constructor for qHttpResponse
     **
     *******************************************************************************/
    public QHttpResponse(HttpResponse httpResponse) throws Exception
+   {
+      setGeneralHttpResponseData(httpResponse);
+      if(this.statusCode == null || this.statusCode != 204)
+      {
+         this.content = EntityUtils.toString(httpResponse.getEntity());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Sets data into this entity from an HttpResponse but doesnt read response data
+    **
+    *******************************************************************************/
+   private void setGeneralHttpResponseData(HttpResponse httpResponse) throws Exception
    {
       this.headerList = Arrays.asList(httpResponse.getAllHeaders());
       if(httpResponse.getStatusLine() != null)
@@ -69,11 +106,6 @@ public class QHttpResponse
             this.statusProtocolVersion = httpResponse.getStatusLine().getProtocolVersion().toString();
          }
       }
-
-      if(this.statusCode == null || this.statusCode != 204)
-      {
-         this.content = EntityUtils.toString(httpResponse.getEntity());
-      }
    }
 
 
@@ -242,4 +274,35 @@ public class QHttpResponse
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for contentBytes
+    *******************************************************************************/
+   public byte[] getContentBytes()
+   {
+      return (this.contentBytes);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for contentBytes
+    *******************************************************************************/
+   public void setContentBytes(byte[] contentBytes)
+   {
+      this.contentBytes = contentBytes;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for contentBytes
+    *******************************************************************************/
+   public QHttpResponse withContentBytes(byte[] contentBytes)
+   {
+      this.contentBytes = contentBytes;
+      return (this);
+   }
+
 }

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/AuthorizationType.java

@@ -33,5 +33,6 @@ public enum AuthorizationType
    BASIC_AUTH_USERNAME_PASSWORD,
    CUSTOM,
    OAUTH2,
+   NONE,
    API_KEY_QUERY_PARAM,
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java

@@ -59,9 +59,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSett
 import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil;
 import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
 import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
@@ -124,10 +126,19 @@ public abstract class AbstractBaseFilesystemAction<FILE>
     *******************************************************************************/
    public abstract InputStream readFile(FILE file) throws IOException;
 
+   /***************************************************************************
+    ** Legacy signature for this method - before table & record params were added.
+    ***************************************************************************/
+   @Deprecated(since = "call the overload that takes table and record")
+   public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
+   {
+      writeFile(backend, null, null, path, contents);
+   }
+
    /*******************************************************************************
     ** Write a file - to be implemented in module-specific subclasses.
     *******************************************************************************/
-   public abstract void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException;
+   public abstract void writeFile(QBackendMetaData backend, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException;
 
    /*******************************************************************************
     ** Get a string that represents the full path to a file.
@@ -287,8 +298,23 @@ public abstract class AbstractBaseFilesystemAction<FILE>
 
          QueryOutput queryOutput = new QueryOutput(queryInput);
 
-         String     requestedPath = null;
-         List<FILE> files         = listFiles(table, queryInput.getBackend(), requestedPath);
+         String requestedPath = null;
+
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if this is a query for a single file name, then get that file name in the requestedPath param for the listFiles call //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(queryInput.getFilter() != null)
+         {
+            for(QFilterCriteria criteria : CollectionUtils.nonNullList(queryInput.getFilter().getCriteria()))
+            {
+               if(criteria.getFieldName().equals(tableDetails.getFileNameFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS))
+               {
+                  requestedPath = ValueUtils.getValueAsString(criteria.getValues().get(0));
+               }
+            }
+         }
+
+         List<FILE> files = listFiles(table, queryInput.getBackend(), requestedPath);
 
          switch(tableDetails.getCardinality())
          {
@@ -632,7 +658,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
                try
                {
                   String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
-                  writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
+                  writeFile(backend, table, record, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
                   record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
                   output.addRecord(record);
                }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java

@@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -204,7 +205,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
     ** Write a file - to be implemented in module-specific subclasses.
     *******************************************************************************/
    @Override
-   public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
+   public void writeFile(QBackendMetaData backend, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException
    {
       FileUtils.writeByteArrayToFile(new File(path), contents);
    }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java

@@ -363,7 +363,7 @@ public class FilesystemImporterStep implements BackendStep
       path = AbstractBaseFilesystemAction.stripDuplicatedSlashes(path);
 
       LOG.info("Archiving file", logPair("path", path), logPair("archiveBackendName", archiveBackend.getName()), logPair("archiveTableName", archiveTable.getName()));
-      archiveActionBase.writeFile(archiveBackend, path, bytes);
+      archiveActionBase.writeFile(archiveBackend, archiveTable, null, path, bytes);
 
       return (path);
    }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java

@@ -111,10 +111,10 @@ public class FilesystemSyncStep implements BackendStep
                   byte[] bytes = inputStream.readAllBytes();
 
                   String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
-                  archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
+                  archiveActionBase.writeFile(archiveBackend, archiveTable, null, archivePath + File.separator + sourceFileName, bytes);
 
                   String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
-                  processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
+                  processingActionBase.writeFile(processingBackend, processingTable, null, processingPath + File.separator + sourceFileName, bytes);
                   syncedFileCount++;
 
                   if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java

@@ -24,8 +24,10 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URLConnection;
 import java.time.Instant;
 import java.util.List;
+import java.util.Objects;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.services.s3.AmazonS3;
@@ -34,6 +36,7 @@ import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -42,6 +45,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFile
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
 import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
@@ -195,14 +199,27 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
     ** Write a file - to be implemented in module-specific subclasses.
     *******************************************************************************/
    @Override
-   public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException
+   public void writeFile(QBackendMetaData backendMetaData, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException
    {
       String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName();
 
       try
       {
          path = stripLeadingSlash(stripDuplicatedSlashes(path));
-         getS3Utils().writeFile(bucketName, path, contents);
+         String contentType = null;
+
+         if(table.getBackendDetails() instanceof S3TableBackendDetails s3TableBackendDetails)
+         {
+            contentType = switch(Objects.requireNonNullElse(s3TableBackendDetails.getContentTypeStrategy(), S3TableBackendDetails.ContentTypeStrategy.NONE))
+            {
+               case BASED_ON_FILE_NAME -> URLConnection.guessContentTypeFromName(path);
+               case FROM_FIELD -> record == null ? null : record.getValueString(s3TableBackendDetails.getContentTypeFieldName());
+               case HARDCODED -> s3TableBackendDetails.getHardcodedContentType();
+               case NONE -> null;
+            };
+         }
+
+         getS3Utils().writeFile(bucketName, path, contents, contentType);
       }
       catch(Exception e)
       {
@@ -277,5 +294,4 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
       getS3Utils().moveObject(bucketName, source, destination);
    }
 
-
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3TableBackendDetails.java

@@ -22,6 +22,11 @@
 package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
 
 
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
 import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
 
@@ -31,6 +36,21 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
  *******************************************************************************/
 public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails
 {
+   private ContentTypeStrategy contentTypeStrategy = ContentTypeStrategy.NONE;
+   private String contentTypeFieldName;
+   private String hardcodedContentType;
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public enum ContentTypeStrategy
+   {
+      BASED_ON_FILE_NAME,
+      FROM_FIELD,
+      HARDCODED,
+      NONE
+   }
 
 
 
@@ -43,4 +63,135 @@ public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails
       setBackendType(S3BackendModule.class);
    }
 
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void validate(QInstance qInstance, QTableMetaData table, QInstanceValidator qInstanceValidator)
+   {
+      super.validate(qInstance, table, qInstanceValidator);
+
+      String prefix = "Table " + (table == null ? "null" : table.getName()) + " backend details - ";
+      switch (Objects.requireNonNullElse(contentTypeStrategy, ContentTypeStrategy.NONE))
+      {
+         case FROM_FIELD ->
+         {
+            qInstanceValidator.assertCondition(!StringUtils.hasContent(hardcodedContentType), prefix + "hardcodedContentType should not be set when contentTypeStrategy is " + contentTypeStrategy);
+
+            if(table != null && qInstanceValidator.assertCondition(StringUtils.hasContent(contentTypeFieldName), prefix + "contentTypeFieldName must be set when contentTypeStrategy is " + contentTypeStrategy))
+            {
+               qInstanceValidator.assertCondition(table.getFields().containsKey(contentTypeFieldName), prefix + "contentTypeFieldName must be a valid field name in the table");
+            }
+         }
+         case HARDCODED ->
+         {
+            qInstanceValidator.assertCondition(!StringUtils.hasContent(contentTypeFieldName), prefix + "contentTypeFieldName should not be set when contentTypeStrategy is " + contentTypeStrategy);
+            qInstanceValidator.assertCondition(StringUtils.hasContent(hardcodedContentType), prefix + "hardcodedContentType must be set when contentTypeStrategy is " + contentTypeStrategy);
+         }
+         case BASED_ON_FILE_NAME, NONE ->
+         {
+            qInstanceValidator.assertCondition(!StringUtils.hasContent(contentTypeFieldName), prefix + "contentTypeFieldName should not be set when contentTypeStrategy is " + contentTypeStrategy);
+            qInstanceValidator.assertCondition(!StringUtils.hasContent(hardcodedContentType), prefix + "hardcodedContentType should not be set when contentTypeStrategy is " + contentTypeStrategy);
+         }
+         default ->
+         {
+            throw new IllegalStateException("Unexpected value: " + contentTypeStrategy);
+         }
+      }
+   }
+
+   /*******************************************************************************
+    ** Getter for contentTypeStrategy
+    *******************************************************************************/
+   public ContentTypeStrategy getContentTypeStrategy()
+   {
+      return (this.contentTypeStrategy);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for contentTypeStrategy
+    *******************************************************************************/
+   public void setContentTypeStrategy(ContentTypeStrategy contentTypeStrategy)
+   {
+      this.contentTypeStrategy = contentTypeStrategy;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for contentTypeStrategy
+    *******************************************************************************/
+   public S3TableBackendDetails withContentTypeStrategy(ContentTypeStrategy contentTypeStrategy)
+   {
+      this.contentTypeStrategy = contentTypeStrategy;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for contentTypeFieldName
+    *******************************************************************************/
+   public String getContentTypeFieldName()
+   {
+      return (this.contentTypeFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for contentTypeFieldName
+    *******************************************************************************/
+   public void setContentTypeFieldName(String contentTypeFieldName)
+   {
+      this.contentTypeFieldName = contentTypeFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for contentTypeFieldName
+    *******************************************************************************/
+   public S3TableBackendDetails withContentTypeFieldName(String contentTypeFieldName)
+   {
+      this.contentTypeFieldName = contentTypeFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for hardcodedContentType
+    *******************************************************************************/
+   public String getHardcodedContentType()
+   {
+      return (this.hardcodedContentType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for hardcodedContentType
+    *******************************************************************************/
+   public void setHardcodedContentType(String hardcodedContentType)
+   {
+      this.hardcodedContentType = hardcodedContentType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for hardcodedContentType
+    *******************************************************************************/
+   public S3TableBackendDetails withHardcodedContentType(String hardcodedContentType)
+   {
+      this.hardcodedContentType = hardcodedContentType;
+      return (this);
+   }
+
+
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.net.URI;
+import java.net.URLEncoder;
 import java.nio.file.FileSystems;
 import java.nio.file.Path;
 import java.nio.file.PathMatcher;
@@ -175,7 +176,7 @@ public class S3Utils
             ///////////////////////////////////////////
             // skip files that do not match the glob //
             ///////////////////////////////////////////
-            if(!pathMatcher.matches(Path.of(URI.create("file:///" + key))))
+            if(!pathMatcher.matches(Path.of(URI.create("file:///" + URLEncoder.encode(key)))))
             {
                // LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob);
                continue;
@@ -204,10 +205,11 @@ public class S3Utils
    /*******************************************************************************
     ** Write a file
     *******************************************************************************/
-   public void writeFile(String bucket, String key, byte[] contents)
+   public void writeFile(String bucket, String key, byte[] contents, String contentType)
    {
       ObjectMetadata objectMetadata = new ObjectMetadata();
       objectMetadata.setContentLength(contents.length);
+      objectMetadata.setContentType(contentType);
       getAmazonS3().putObject(bucket, key, new ByteArrayInputStream(contents), objectMetadata);
    }
 

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java

@@ -325,20 +325,51 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
             fullPath = "." + fullPath;
          }
 
-         for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath))
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // in case we were asked to list a single file name, make sure we don't put a slash on the end (which wouldn't be found) //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(fullPath.endsWith("/"))
          {
-            if(".".equals(dirEntry.getFilename()) || "..".equals(dirEntry.getFilename()))
-            {
-               continue;
-            }
+            fullPath = fullPath.substring(0, fullPath.length() - 1);
+         }
 
-            if(dirEntry.getAttributes().isDirectory())
-            {
-               // todo - recursive??
-               continue;
-            }
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // make a 'stat' call, to find out if the path is found, and if it is, if it describes a single file, or a directory //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         SftpClient.Attributes stat = sftpClient.stat(fullPath);
+         if(stat == null)
+         {
+            return (rs);
+         }
+         else if(stat.isRegularFile())
+         {
+            ///////////////////////////////////////////////////////////////////////////
+            // split up the fullPath into its directory prefix, and file name suffix //
+            ///////////////////////////////////////////////////////////////////////////
+            int    lastSlashIndex = fullPath.lastIndexOf("/");
+            String directory      = lastSlashIndex == -1 ? "./" : fullPath.substring(0, lastSlashIndex);
+            String fileBaseName   = lastSlashIndex == -1 ? fullPath : fullPath.substring(lastSlashIndex + 1);
 
-            rs.add(new SFTPDirEntryWithPath(fullPath, dirEntry));
+            SftpClient.DirEntry dirEntry = new SftpClient.DirEntry(fileBaseName, fullPath, stat);
+            rs.add(new SFTPDirEntryWithPath(directory, dirEntry));
+         }
+         else if(stat.isDirectory())
+         {
+            for(SftpClient.DirEntry dirEntry : sftpClient.readEntries(fullPath))
+            {
+               if(".".equals(dirEntry.getFilename()) || "..".equals(dirEntry.getFilename()))
+               {
+                  continue;
+               }
+
+               if(dirEntry.getAttributes().isDirectory())
+               {
+                  // todo - recursive??
+                  continue;
+               }
+
+               rs.add(new SFTPDirEntryWithPath(fullPath, dirEntry));
+            }
          }
 
          return (rs);
@@ -372,7 +403,7 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
     **
     ***************************************************************************/
    @Override
-   public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
+   public void writeFile(QBackendMetaData backend, QTableMetaData table, QRecord record, String path, byte[] contents) throws IOException
    {
       sftpClient.put(new ByteArrayInputStream(contents), path);
    }

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java

@@ -362,6 +362,7 @@ public class TestUtils
          .withField(new QFieldMetaData("fileName", QFieldType.STRING))
          .withField(new QFieldMetaData("contents", QFieldType.BLOB))
          .withBackendDetails(new S3TableBackendDetails()
+            .withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.BASED_ON_FILE_NAME)
             .withBasePath("blobs")
             .withCardinality(Cardinality.ONE)
             .withFileNameFieldName("fileName")

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java

@@ -229,7 +229,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
       AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
       String           fullPath   = actionBase.getFullBasePath(table, backend);
 
-      actionBase.writeFile(backend, fullPath + "/" + name, content.getBytes());
+      actionBase.writeFile(backend, table, null, fullPath + "/" + name, content.getBytes());
    }
 
 

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java

@@ -66,7 +66,7 @@ public class BaseS3Test extends BaseTest
       amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
       amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-1.txt", "Hello, Blob");
       amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-2.txt", "Hi, Bob");
-      amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-3.md", "# Hi, MD");
+      amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB 3.md", "# Hi, MD"); // this one, with a space in the name, tripped up listObjectsInBucketMatchingGlob's path matching at one time
 
       amazonS3.createBucket(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND);
       amazonS3.putObject(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND, "BLOB-1.txt", "Hello, Blob");

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertActionTest.java

@@ -25,7 +25,9 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
+import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.S3Object;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
@@ -34,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
+import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.NotImplementedException;
 import org.junit.jupiter.api.Test;
@@ -54,18 +57,15 @@ public class S3InsertActionTest extends BaseS3Test
    @Test
    public void testCardinalityOne() throws QException, IOException
    {
-      QInstance qInstance = TestUtils.defineInstance();
-
       InsertInput insertInput = new InsertInput();
       insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
       insertInput.setRecords(List.of(
-         new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
-      ));
+         new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")));
 
       S3InsertAction insertAction = new S3InsertAction();
       insertAction.setS3Utils(getS3Utils());
-
       InsertOutput insertOutput = insertAction.execute(insertInput);
+
       assertThat(insertOutput.getRecords())
          .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
 
@@ -73,6 +73,65 @@ public class S3InsertActionTest extends BaseS3Test
       S3Object     object   = getAmazonS3().getObject(BUCKET_NAME, fullPath);
       List<String> lines    = IOUtils.readLines(object.getObjectContent(), StandardCharsets.UTF_8);
       assertEquals("Hi, Bob.", lines.get(0));
+
+      ObjectMetadata objectMetadata = object.getObjectMetadata();
+      assertEquals("text/plain", objectMetadata.getContentType());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testContentTypeFromField() throws QException
+   {
+      ((S3TableBackendDetails) QContext.getQInstance().getTable(TestUtils.TABLE_NAME_BLOB_S3)
+         .getBackendDetails())
+         .withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.FROM_FIELD)
+         .withContentTypeFieldName("contentType");
+
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("fileName", "file2.txt").withValue("contentType", "myContentType/fake").withValue("contents", "Hi, Bob.")));
+
+      S3InsertAction insertAction = new S3InsertAction();
+      insertAction.setS3Utils(getS3Utils());
+      InsertOutput insertOutput = insertAction.execute(insertInput);
+
+      String   fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
+      S3Object object   = getAmazonS3().getObject(BUCKET_NAME, fullPath);
+      ObjectMetadata objectMetadata = object.getObjectMetadata();
+      assertEquals("myContentType/fake", objectMetadata.getContentType());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testContentTypeHardcoded() throws QException
+   {
+      ((S3TableBackendDetails) QContext.getQInstance().getTable(TestUtils.TABLE_NAME_BLOB_S3)
+         .getBackendDetails())
+         .withContentTypeStrategy(S3TableBackendDetails.ContentTypeStrategy.HARDCODED)
+         .withHardcodedContentType("your-content-type");
+
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")));
+
+      S3InsertAction insertAction = new S3InsertAction();
+      insertAction.setS3Utils(getS3Utils());
+      InsertOutput insertOutput = insertAction.execute(insertInput);
+
+      String   fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
+      S3Object object   = getAmazonS3().getObject(BUCKET_NAME, fullPath);
+      ObjectMetadata objectMetadata = object.getObjectMetadata();
+      assertEquals("your-content-type", objectMetadata.getContentType());
    }