updates to check versions on process query params

Feature/string utils safe equals ignore case

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfAction.java

@@ -35,6 +35,7 @@ import com.openhtmltopdf.pdfboxout.PdfBoxFontResolver;
 import com.openhtmltopdf.pdfboxout.PdfBoxRenderer;
 import com.openhtmltopdf.pdfboxout.PdfRendererBuilder;
 import org.jsoup.Jsoup;
+import org.jsoup.helper.W3CDom;
 import org.jsoup.nodes.Document;
 
 
@@ -67,6 +68,7 @@ public class ConvertHtmlToPdfAction extends AbstractQActionFunction<ConvertHtmlT
          //////////////////////////////////////////////////////////////////
          Document document = Jsoup.parse(input.getHtml());
          document.outputSettings().syntax(Document.OutputSettings.Syntax.xml);
+         org.w3c.dom.Document w3cDoc = new W3CDom().fromJsoup(document);
 
          //////////////////////////////
          // convert the XHTML to PDF //
@@ -74,7 +76,7 @@ public class ConvertHtmlToPdfAction extends AbstractQActionFunction<ConvertHtmlT
          PdfRendererBuilder builder = new PdfRendererBuilder();
          builder.toStream(input.getOutputStream());
          builder.useFastMode();
-         builder.withHtmlContent(document.html(), input.getBasePath() == null ? "./" : input.getBasePath().toUri().toString());
+         builder.withW3cDocument(w3cDoc, input.getBasePath() == null ? "./" : input.getBasePath().toUri().toString());
 
          try(PdfBoxRenderer pdfBoxRenderer = builder.buildPdfRenderer())
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/BasicCustomPossibleValueProvider.java

@@ -0,0 +1,91 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.values;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
+
+
+/*******************************************************************************
+ ** Basic implementation of a possible value provider, for where there's a limited
+ ** set of possible source objects - so you just have to define how to make one
+ ** PV from a source object, how to list all of the source objects, and how to
+ ** look up a PV from an id.
+ *******************************************************************************/
+public abstract class BasicCustomPossibleValueProvider<S, ID extends Serializable> implements QCustomPossibleValueProvider<ID>
+{
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected abstract QPossibleValue<ID> makePossibleValue(S sourceObject);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected abstract S getSourceObject(Serializable id);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected abstract List<S> getAllSourceObjects();
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public QPossibleValue<ID> getPossibleValue(Serializable idValue)
+   {
+      S sourceObject = getSourceObject(idValue);
+      if(sourceObject == null)
+      {
+         return (null);
+      }
+
+      return makePossibleValue(sourceObject);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public List<QPossibleValue<ID>> search(SearchPossibleValueSourceInput input) throws QException
+   {
+      List<QPossibleValue<ID>> allPossibleValues = new ArrayList<>();
+      List<S>                  allSourceObjects  = getAllSourceObjects();
+      for(S sourceObject : allSourceObjects)
+      {
+         allPossibleValues.add(makePossibleValue(sourceObject));
+      }
+
+      return completeCustomPVSSearch(input, allPossibleValues);
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java

@@ -66,6 +66,7 @@ public enum WidgetType
    // record view/edit widgets //
    //////////////////////////////
    CHILD_RECORD_LIST("childRecordList"),
+   CUSTOM_COMPONENT("customComponent"),
    DYNAMIC_FORM("dynamicForm"),
    DATA_BAG_VIEWER("dataBagViewer"),
    PIVOT_TABLE_SETUP("pivotTableSetup"),

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/WhiteSpaceBehavior.java

@@ -0,0 +1,174 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.fields;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ ** Field behavior that changes the whitespace of string values.
+ *******************************************************************************/
+public enum WhiteSpaceBehavior implements FieldBehavior<WhiteSpaceBehavior>, FieldBehaviorForFrontend, FieldFilterBehavior<WhiteSpaceBehavior>
+{
+   NONE(null),
+   REMOVE_ALL_WHITESPACE((String s) -> s.chars().filter(c -> !Character.isWhitespace(c)).collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append).toString()),
+   TRIM((String s) -> s.trim()),
+   TRIM_LEFT((String s) -> s.stripLeading()),
+   TRIM_RIGHT((String s) -> s.stripTrailing());
+
+
+   private final Function<String, String> function;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   WhiteSpaceBehavior(Function<String, String> function)
+   {
+      this.function = function;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public WhiteSpaceBehavior getDefault()
+   {
+      return (NONE);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
+   {
+      if(this.equals(NONE))
+      {
+         return;
+      }
+
+      switch(this)
+      {
+         case REMOVE_ALL_WHITESPACE, TRIM, TRIM_LEFT, TRIM_RIGHT -> applyFunction(recordList, table, field);
+         default -> throw new IllegalStateException("Unexpected enum value: " + this);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void applyFunction(List<QRecord> recordList, QTableMetaData table, QFieldMetaData field)
+   {
+      String fieldName = field.getName();
+      for(QRecord record : CollectionUtils.nonNullList(recordList))
+      {
+         String value = record.getValueString(fieldName);
+         if(value != null && function != null)
+         {
+            record.setValue(fieldName, function.apply(value));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public Serializable applyToFilterCriteriaValue(Serializable value, QInstance instance, QTableMetaData table, QFieldMetaData field)
+   {
+      if(this.equals(NONE) || function == null)
+      {
+         return (value);
+      }
+
+      if(value instanceof String s)
+      {
+         String newValue = function.apply(s);
+         if(!Objects.equals(value, newValue))
+         {
+            return (newValue);
+         }
+      }
+
+      return (value);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean allowMultipleBehaviorsOfThisType()
+   {
+      return (false);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<String> validateBehaviorConfiguration(QTableMetaData tableMetaData, QFieldMetaData fieldMetaData)
+   {
+      if(this == NONE)
+      {
+         return Collections.emptyList();
+      }
+
+      List<String> errors      = new ArrayList<>();
+      String       errorSuffix = " field [" + fieldMetaData.getName() + "] in table [" + tableMetaData.getName() + "]";
+
+      if(fieldMetaData.getType() != null)
+      {
+         if(!fieldMetaData.getType().isStringLike())
+         {
+            errors.add("A WhiteSpaceBehavior was a applied to a non-String-like field:" + errorSuffix);
+         }
+      }
+
+      return (errors);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java

@@ -27,11 +27,9 @@ import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
-import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
+import com.kingsrook.qqq.backend.core.actions.values.BasicCustomPossibleValueProvider;
 import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
-import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
@@ -40,26 +38,16 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
  ** possible-value source provider for the `Tables` PVS - a list of all tables
  ** in an application/qInstance.
  *******************************************************************************/
-public class TablesCustomPossibleValueProvider implements QCustomPossibleValueProvider<String>
+public class TablesCustomPossibleValueProvider extends BasicCustomPossibleValueProvider<QTableMetaData, String>
 {
 
    /***************************************************************************
     **
     ***************************************************************************/
    @Override
-   public QPossibleValue<String> getPossibleValue(Serializable idValue)
+   protected QPossibleValue<String> makePossibleValue(QTableMetaData sourceObject)
    {
-      QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(idValue));
-      if(table != null && !table.getIsHidden())
-      {
-         PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table);
-         if(PermissionCheckResult.ALLOW.equals(permissionCheckResult))
-         {
-            return (new QPossibleValue<>(table.getName(), table.getLabel()));
-         }
-      }
-
-      return null;
+      return (new QPossibleValue<>(sourceObject.getName(), sourceObject.getLabel()));
    }
 
 
@@ -68,22 +56,54 @@ public class TablesCustomPossibleValueProvider implements QCustomPossibleValuePr
     **
     ***************************************************************************/
    @Override
-   public List<QPossibleValue<String>> search(SearchPossibleValueSourceInput input) throws QException
+   protected QTableMetaData getSourceObject(Serializable id)
    {
-      /////////////////////////////////////////////////////////////////////////////////////
-      // build all of the possible values (note, will be filtered by user's permissions) //
-      /////////////////////////////////////////////////////////////////////////////////////
-      List<QPossibleValue<String>> allPossibleValues = new ArrayList<>();
+      QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(id));
+      return isTableAllowed(table) ? table : null;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   protected List<QTableMetaData> getAllSourceObjects()
+   {
+      ArrayList<QTableMetaData> rs = new ArrayList<>();
       for(QTableMetaData table : QContext.getQInstance().getTables().values())
       {
-         QPossibleValue<String> possibleValue = getPossibleValue(table.getName());
-         if(possibleValue != null)
+         if(isTableAllowed(table))
          {
-            allPossibleValues.add(possibleValue);
+            rs.add(table);
          }
       }
+      return rs;
+   }
 
-      return completeCustomPVSSearch(input, allPossibleValues);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private boolean isTableAllowed(QTableMetaData table)
+   {
+      if(table == null)
+      {
+         return (false);
+      }
+
+      if(table.getIsHidden())
+      {
+         return (false);
+      }
+
+      PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table);
+      if(!PermissionCheckResult.ALLOW.equals(permissionCheckResult))
+      {
+         return (false);
+      }
+
+      return (true);
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables;
 
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
@@ -45,6 +46,7 @@ public class TablesPossibleValueSourceMetaDataProvider
    {
       QPossibleValueSource possibleValueSource = new QPossibleValueSource()
          .withName(NAME)
+         .withIdType(QFieldType.STRING)
          .withType(QPossibleValueSourceType.CUSTOM)
          .withCustomCodeReference(new QCodeReference(TablesCustomPossibleValueProvider.class))
          .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java

@@ -55,8 +55,6 @@ public class BulkInsertExtractStep extends AbstractExtractStep
    @Override
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
-      runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput));
-
       int rowsAdded     = 0;
       int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java

@@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
+import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
@@ -77,19 +78,77 @@ public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSumm
 
       QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getValueString("tableName"));
 
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // the transform step builds summary lines that it predicts will insert successfully.      //
+      // but those lines don't have ids, which we'd like to have (e.g., for a process trace that //
+      // might link to the built record).  also, it's possible that there was a fail that only   //
+      // happened in the actual insert, so, basically, re-do the summary here                    //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      BulkInsertTransformStep transformStep = (BulkInsertTransformStep) getTransformStep();
+      ProcessSummaryLine      okSummary     = transformStep.okSummary;
+      okSummary.setCount(0);
+      okSummary.setPrimaryKeys(new ArrayList<>());
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // but - since errors from the transform step don't even make it through to us here in the load step,   //
+      // do re-use the ProcessSummaryWarningsAndErrorsRollup from transform step as follows:                  //
+      // clear out its warnings - we'll completely rebuild them here (with primary keys)                      //
+      // and add new error lines, e.g., in case of errors that only happened past the validation if possible. //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////
+      ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = transformStep.processSummaryWarningsAndErrorsRollup;
+      processSummaryWarningsAndErrorsRollup.resetWarnings();
+
       List<QRecord> insertedRecords = runBackendStepOutput.getRecords();
       for(QRecord insertedRecord : insertedRecords)
       {
-         if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()))
+         Serializable primaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
+         if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()) && primaryKey != null)
          {
+            /////////////////////////////////////////////////////////////////////////
+            // if the record had no errors, and we have a primary key for it, then //
+            // keep track of the range of primary keys (first and last)            //
+            /////////////////////////////////////////////////////////////////////////
             if(firstInsertedPrimaryKey == null)
             {
-               firstInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
+               firstInsertedPrimaryKey = primaryKey;
             }
 
-            lastInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
+            lastInsertedPrimaryKey = primaryKey;
+
+            if(!CollectionUtils.nullSafeIsEmpty(insertedRecord.getWarnings()))
+            {
+               /////////////////////////////////////////////////////////////////////////////
+               // if there were warnings on the inserted record, put it in a warning line //
+               /////////////////////////////////////////////////////////////////////////////
+               String message = insertedRecord.getWarnings().get(0).getMessage();
+               processSummaryWarningsAndErrorsRollup.addWarning(message, primaryKey);
+            }
+            else
+            {
+               ////////////////////////////////////////////////////////////////////////
+               // if no warnings for the inserted record, then put it in the OK line //
+               ////////////////////////////////////////////////////////////////////////
+               okSummary.incrementCountAndAddPrimaryKey(primaryKey);
+            }
+         }
+         else
+         {
+            //////////////////////////////////////////////////////////////////////
+            // else if there were errors or no primary key, build an error line //
+            //////////////////////////////////////////////////////////////////////
+            String message = "Failed to insert";
+            if(!CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()))
+            {
+               //////////////////////////////////////////////////////////
+               // use the error message from the record if we have one //
+               //////////////////////////////////////////////////////////
+               message = insertedRecord.getErrors().get(0).getMessage();
+            }
+            processSummaryWarningsAndErrorsRollup.addError(message, primaryKey);
          }
       }
+
+      okSummary.pickMessage(true);
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java

@@ -52,6 +52,11 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep
    @Override
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
+      ////////////////////////////////////////////////////////////////////////////////////////
+      // for headless-bulk load (e.g., sftp import), set up the process tracer's key record //
+      ////////////////////////////////////////////////////////////////////////////////////////
+      runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput));
+
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // if user has come back here, clear out file (else the storageInput object that it is comes to the frontend, which isn't what we want!) //
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java

@@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mode
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.lang3.BooleanUtils;
 
@@ -77,10 +78,14 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
             //////////////////////////////////////////////////////////////////////////////
             if(savedBulkLoadProfileRecord == null)
             {
-               throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load"));
+               throw (new QUserFacingException("Did not receive a Bulk Load Profile record as input.  Unable to perform headless bulk load"));
             }
 
             SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord);
+            if(!StringUtils.hasContent(savedBulkLoadProfile.getMappingJson()))
+            {
+               throw (new QUserFacingException("Bulk Load Profile record's Mapping is empty.  Unable to perform headless bulk load"));
+            }
 
             try
             {
@@ -88,7 +93,7 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
             }
             catch(Exception e)
             {
-               throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e));
+               throw (new QUserFacingException("Error processing Bulk Load Profile record.  Unable to perform headless bulk load", e));
             }
          }
          else
@@ -240,6 +245,11 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
             }
          }
       }
+      catch(QUserFacingException ufe)
+      {
+         LOG.warn("User-facing error in bulk insert receive mapping", ufe);
+         throw ufe;
+      }
       catch(Exception e)
       {
          LOG.warn("Error in bulk insert receive mapping", e);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java

@@ -75,9 +75,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
  *******************************************************************************/
 public class BulkInsertTransformStep extends AbstractTransformStep
 {
-   private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
+   ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
 
-   private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted")
+   ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted")
       .withDoReplaceSingletonCountLinesWithSuffixOnly(false);
 
    private ListingHash<String, RowValue> errorToExampleRowValueMap = new ListingHash<>();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java

@@ -195,7 +195,7 @@ public class ProcessSummaryWarningsAndErrorsRollup
             {
                if(otherWarningsSummary == null)
                {
-                  otherWarningsSummary = new ProcessSummaryLine(Status.WARNING).withMessageSuffix("records had an other warning.");
+                  otherWarningsSummary = buildOtherWarningsSummary();
                }
                processSummaryLine = otherWarningsSummary;
             }
@@ -214,6 +214,27 @@ public class ProcessSummaryWarningsAndErrorsRollup
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static ProcessSummaryLine buildOtherWarningsSummary()
+   {
+      return new ProcessSummaryLine(Status.WARNING).withMessageSuffix("records had an other warning.");
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public void resetWarnings()
+   {
+      warningSummaries.clear();
+      otherWarningsSummary = buildOtherWarningsSummary();
+   }
+
+
+
    /*******************************************************************************
     ** Wrapper around AlphaNumericComparator for ProcessSummaryLineInterface that
     ** extracts string messages out.

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.utils;
 
 
 import java.io.Serializable;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -51,13 +52,14 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
  *******************************************************************************/
 public class RecordLookupHelper
 {
-   private Map<String, Map<Serializable, QRecord>> recordMaps = new HashMap<>();
+   private Map<String, Map<Serializable, QRecord>> recordMaps;
 
-   private Map<String, Map<Map<String, Serializable>, QRecord>> uniqueKeyMaps = new HashMap<>();
+   private Map<String, Map<Map<String, Serializable>, QRecord>> uniqueKeyMaps;
 
-   private Set<String> preloadedKeys = new HashSet<>();
+   private Set<String> preloadedKeys;
 
-   private Set<Pair<String, String>> disallowedOneOffLookups = new HashSet<>();
+   private Set<Pair<String, String>> disallowedOneOffLookups;
+   private boolean                   useSynchronizedCollections;
 
 
 
@@ -67,6 +69,33 @@ public class RecordLookupHelper
     *******************************************************************************/
    public RecordLookupHelper()
    {
+      this(false);
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public RecordLookupHelper(boolean useSynchronizedCollections)
+   {
+      this.useSynchronizedCollections = useSynchronizedCollections;
+
+      if(useSynchronizedCollections)
+      {
+         recordMaps = Collections.synchronizedMap(new HashMap<>());
+         uniqueKeyMaps = Collections.synchronizedMap(new HashMap<>());
+         preloadedKeys = Collections.synchronizedSet(new HashSet<>());
+         disallowedOneOffLookups = Collections.synchronizedSet(new HashSet<>());
+      }
+      else
+      {
+         recordMaps = new HashMap<>();
+         uniqueKeyMaps = new HashMap<>();
+         preloadedKeys = new HashSet<>();
+         disallowedOneOffLookups = new HashSet<>();
+      }
    }
 
 
@@ -77,7 +106,7 @@ public class RecordLookupHelper
    public QRecord getRecordByUniqueKey(String tableName, Map<String, Serializable> uniqueKey) throws QException
    {
       String                                  mapKey    = tableName + "." + uniqueKey.keySet().stream().sorted().collect(Collectors.joining(","));
-      Map<Map<String, Serializable>, QRecord> recordMap = uniqueKeyMaps.computeIfAbsent(mapKey, (k) -> new HashMap<>());
+      Map<Map<String, Serializable>, QRecord> recordMap = uniqueKeyMaps.computeIfAbsent(mapKey, (k) -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
 
       if(!recordMap.containsKey(uniqueKey))
       {
@@ -96,7 +125,7 @@ public class RecordLookupHelper
    public QRecord getRecordByKey(String tableName, String keyFieldName, Serializable key) throws QException
    {
       String                     mapKey    = tableName + "." + keyFieldName;
-      Map<Serializable, QRecord> recordMap = recordMaps.computeIfAbsent(mapKey, (k) -> new HashMap<>());
+      Map<Serializable, QRecord> recordMap = recordMaps.computeIfAbsent(mapKey, (k) -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
 
       ////////////////////////////////////////////////////////////
       // make sure we have they key object in the expected type //
@@ -150,7 +179,7 @@ public class RecordLookupHelper
    public void preloadRecords(String tableName, String keyFieldName, QQueryFilter filter) throws QException
    {
       String                     mapKey   = tableName + "." + keyFieldName;
-      Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>());
+      Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
       tableMap.putAll(GeneralProcessUtils.loadTableToMap(tableName, keyFieldName, filter));
    }
 
@@ -170,7 +199,7 @@ public class RecordLookupHelper
       }
 
       String                     mapKey   = tableName + "." + keyFieldName;
-      Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>());
+      Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
 
       QQueryFilter filter = new QQueryFilter(new QFilterCriteria(keyFieldName, QCriteriaOperator.IN, inList));
       tableMap.putAll(GeneralProcessUtils.loadTableToMap(tableName, keyFieldName, filter));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.state;
 import java.io.Serializable;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
@@ -58,7 +59,7 @@ public class InMemoryStateProvider implements StateProviderInterface
     *******************************************************************************/
    private InMemoryStateProvider()
    {
-      this.map = new HashMap<>();
+      this.map = Collections.synchronizedMap(new HashMap<>());
 
       ///////////////////////////////////////////////////////////
       // Start a single thread executor to handle the cleaning //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java

@@ -489,6 +489,24 @@ public class StringUtils
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static boolean safeEqualsIgnoreCase(String a, String b)
+   {
+      if(a == null && b == null)
+      {
+         return true;
+      }
+      if(a == null || b == null)
+      {
+         return false;
+      }
+      return (a.equalsIgnoreCase(b));
+   }
+
+
+
    /***************************************************************************
     **
     ***************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java

@@ -79,6 +79,7 @@ class ConvertHtmlToPdfActionTest extends BaseTest
                      </h1>
                      <div class="myclass">
                          <p>This is a test of converting HTML to PDF!!</p>
+                         <p>This is &nbsp; a line with &bull; some entities &lt;</p>
                          <p style="font-family: SF-Pro; font-size: 24px;">(btw, is this in SF-Pro???)</p>
                      </div>
                  </div>

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java

@@ -185,4 +185,13 @@ public class ProcessSummaryLineInterfaceAssert extends AbstractAssert<ProcessSum
       return (this);
    }
 
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public ProcessSummaryLineInterface getLine()
+   {
+      return actual;
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/WhiteSpaceBehaviorTest.java

@@ -0,0 +1,280 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.fields;
+
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for WhiteSpaceBehavior
+ *******************************************************************************/
+class WhiteSpaceBehaviorTest extends BaseTest
+{
+   public static final String FIELD = "firstName";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testNone()
+   {
+      assertNull(applyToRecord(WhiteSpaceBehavior.NONE, new QRecord(), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertNull(applyToRecord(WhiteSpaceBehavior.NONE, new QRecord().withValue(FIELD, null), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertEquals("John", applyToRecord(WhiteSpaceBehavior.NONE, new QRecord().withValue(FIELD, "John"), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+
+      assertEquals(ListBuilder.of("J.   ohn", null, "Jane\n"), applyToRecords(WhiteSpaceBehavior.NONE, List.of(
+            new QRecord().withValue(FIELD, "J.   ohn"),
+            new QRecord(),
+            new QRecord().withValue(FIELD, "Jane\n")),
+         ValueBehaviorApplier.Action.INSERT).stream().map(r -> r.getValueString(FIELD)).toList());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testRemoveWhiteSpace()
+   {
+      assertNull(applyToRecord(WhiteSpaceBehavior.REMOVE_ALL_WHITESPACE, new QRecord(), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertNull(applyToRecord(WhiteSpaceBehavior.REMOVE_ALL_WHITESPACE, new QRecord().withValue(FIELD, null), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertEquals("doobeedoobeedoo", applyToRecord(WhiteSpaceBehavior.REMOVE_ALL_WHITESPACE, new QRecord().withValue(FIELD, "doo bee doo\n bee doo"), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+
+      assertEquals(ListBuilder.of("thisistheway", null, "thatwastheway"), applyToRecords(WhiteSpaceBehavior.REMOVE_ALL_WHITESPACE, List.of(
+            new QRecord().withValue(FIELD, "this is\rthe way   \t"),
+            new QRecord(),
+            new QRecord().withValue(FIELD, "that was the way\n")),
+         ValueBehaviorApplier.Action.INSERT).stream().map(r -> r.getValueString(FIELD)).toList());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTrimWhiteSpace()
+   {
+      assertNull(applyToRecord(WhiteSpaceBehavior.TRIM, new QRecord(), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertNull(applyToRecord(WhiteSpaceBehavior.TRIM, new QRecord().withValue(FIELD, null), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertEquals("doo bee doo\n bee doo", applyToRecord(WhiteSpaceBehavior.TRIM, new QRecord().withValue(FIELD, "    doo bee doo\n bee doo\r \n\n"), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+
+      assertEquals(ListBuilder.of("this is\rthe way", null, "that was the way"), applyToRecords(WhiteSpaceBehavior.TRIM, List.of(
+            new QRecord().withValue(FIELD, "      this is\rthe way   \t"),
+            new QRecord(),
+            new QRecord().withValue(FIELD, "that was the way\n")),
+         ValueBehaviorApplier.Action.INSERT).stream().map(r -> r.getValueString(FIELD)).toList());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTrimLeftWhiteSpace()
+   {
+      assertNull(applyToRecord(WhiteSpaceBehavior.TRIM_LEFT, new QRecord(), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertNull(applyToRecord(WhiteSpaceBehavior.TRIM_LEFT, new QRecord().withValue(FIELD, null), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertEquals("doo bee doo\n bee doo\r \n\n", applyToRecord(WhiteSpaceBehavior.TRIM_LEFT, new QRecord().withValue(FIELD, "    doo bee doo\n bee doo\r \n\n"), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+
+      assertEquals(ListBuilder.of("this is\rthe way   \t", null, "that was the way\n"), applyToRecords(WhiteSpaceBehavior.TRIM_LEFT, List.of(
+            new QRecord().withValue(FIELD, "      this is\rthe way   \t"),
+            new QRecord(),
+            new QRecord().withValue(FIELD, "  \n that was the way\n")),
+         ValueBehaviorApplier.Action.INSERT).stream().map(r -> r.getValueString(FIELD)).toList());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTrimRightWhiteSpace()
+   {
+      assertNull(applyToRecord(WhiteSpaceBehavior.TRIM_RIGHT, new QRecord(), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertNull(applyToRecord(WhiteSpaceBehavior.TRIM_RIGHT, new QRecord().withValue(FIELD, null), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+      assertEquals("    doo bee doo\n bee doo", applyToRecord(WhiteSpaceBehavior.TRIM_RIGHT, new QRecord().withValue(FIELD, "    doo bee doo\n bee doo\r \n\n"), ValueBehaviorApplier.Action.INSERT).getValue(FIELD));
+
+      assertEquals(ListBuilder.of("      this is\rthe way", null, "  \n that was the way"), applyToRecords(WhiteSpaceBehavior.TRIM_RIGHT, List.of(
+            new QRecord().withValue(FIELD, "      this is\rthe way   \t"),
+            new QRecord(),
+            new QRecord().withValue(FIELD, "  \n that was the way\n")),
+         ValueBehaviorApplier.Action.INSERT).stream().map(r -> r.getValueString(FIELD)).toList());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QRecord applyToRecord(WhiteSpaceBehavior behavior, QRecord record, ValueBehaviorApplier.Action action)
+   {
+      return (applyToRecords(behavior, List.of(record), action).get(0));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<QRecord> applyToRecords(WhiteSpaceBehavior behavior, List<QRecord> records, ValueBehaviorApplier.Action action)
+   {
+      QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      behavior.apply(action, records, QContext.getQInstance(), table, table.getField(FIELD));
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testReads() throws QException
+   {
+      TestUtils.insertDefaultShapes(QContext.getQInstance());
+
+      List<QRecord> records = QueryAction.execute(TestUtils.TABLE_NAME_SHAPE, null);
+      assertEquals(Set.of("Triangle", "Square", "Circle"), records.stream().map(r -> r.getValueString("name")).collect(Collectors.toSet()));
+
+      QFieldMetaData field = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_SHAPE).getField("name");
+      field.setBehaviors(Set.of(CaseChangeBehavior.TO_UPPER_CASE));
+
+      records = QueryAction.execute(TestUtils.TABLE_NAME_SHAPE, null);
+      assertEquals(Set.of("TRIANGLE", "SQUARE", "CIRCLE"), records.stream().map(r -> r.getValueString("name")).collect(Collectors.toSet()));
+
+      field.setBehaviors(Set.of(CaseChangeBehavior.TO_LOWER_CASE));
+      assertEquals("triangle", GetAction.execute(TestUtils.TABLE_NAME_SHAPE, 1).getValueString("name"));
+
+      field.setBehaviors(Set.of(CaseChangeBehavior.NONE));
+      assertEquals("Triangle", GetAction.execute(TestUtils.TABLE_NAME_SHAPE, 1).getValueString("name"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testWrites() throws QException
+   {
+      Integer id = 100;
+
+      QFieldMetaData field = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_SHAPE).getField("name");
+      field.setBehaviors(Set.of(CaseChangeBehavior.TO_UPPER_CASE));
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_SHAPE).withRecord(new QRecord().withValue("id", id).withValue("name", "Octagon")));
+
+      //////////////////////////////////////////////////////////////////////////////////
+      // turn off the to-upper-case behavior, so we'll see what was actually inserted //
+      //////////////////////////////////////////////////////////////////////////////////
+      field.setBehaviors(Collections.emptySet());
+      assertEquals("OCTAGON", GetAction.execute(TestUtils.TABLE_NAME_SHAPE, id).getValueString("name"));
+
+      ////////////////////////////////////////////
+      // change to toLowerCase and do an update //
+      ////////////////////////////////////////////
+      field.setBehaviors(Set.of(CaseChangeBehavior.TO_LOWER_CASE));
+      new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_SHAPE).withRecord(new QRecord().withValue("id", id).withValue("name", "Octagon")));
+
+      ////////////////////////////////////////////////////////////////////////////////////
+      // turn off the to-lower-case behavior, so we'll see what was actually updated to //
+      ////////////////////////////////////////////////////////////////////////////////////
+      field.setBehaviors(Collections.emptySet());
+      assertEquals("octagon", GetAction.execute(TestUtils.TABLE_NAME_SHAPE, id).getValueString("name"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFilter()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_SHAPE);
+      QFieldMetaData field     = table.getField("name");
+      field.setBehaviors(Set.of(CaseChangeBehavior.TO_UPPER_CASE));
+      assertEquals("SQUARE", CaseChangeBehavior.TO_UPPER_CASE.applyToFilterCriteriaValue("square", qInstance, table, field));
+
+      field.setBehaviors(Set.of(CaseChangeBehavior.TO_LOWER_CASE));
+      assertEquals("triangle", CaseChangeBehavior.TO_LOWER_CASE.applyToFilterCriteriaValue("Triangle", qInstance, table, field));
+
+      field.setBehaviors(Set.of(CaseChangeBehavior.NONE));
+      assertEquals("Circle", CaseChangeBehavior.NONE.applyToFilterCriteriaValue("Circle", qInstance, table, field));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testValidation()
+   {
+      QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_SHAPE);
+
+      ///////////////////////////////////////////
+      // should be no errors on a string field //
+      ///////////////////////////////////////////
+      assertTrue(CaseChangeBehavior.TO_UPPER_CASE.validateBehaviorConfiguration(table, table.getField("name")).isEmpty());
+
+      //////////////////////////////////////////
+      // should be an error on a number field //
+      //////////////////////////////////////////
+      assertEquals(1, CaseChangeBehavior.TO_LOWER_CASE.validateBehaviorConfiguration(table, table.getField("id")).size());
+
+      /////////////////////////////////////////
+      // NONE should be allowed on any field //
+      /////////////////////////////////////////
+      assertTrue(CaseChangeBehavior.NONE.validateBehaviorConfiguration(table, table.getField("id")).isEmpty());
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java

@@ -29,15 +29,26 @@ import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
 import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
@@ -176,7 +187,15 @@ class BulkInsertFullProcessTest extends BaseTest
       assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class);
       assertThat(runProcessOutput.getException()).isEmpty();
 
-      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id values between 1 and 2");
+      ProcessSummaryLineInterface okLine = ProcessSummaryAssert.assertThat(runProcessOutput)
+         .hasLineWithMessageContaining("Person Memory records were inserted")
+         .hasStatus(Status.OK)
+         .hasCount(2)
+         .getLine();
+      assertEquals(List.of(1, 2), ((ProcessSummaryLine) okLine).getPrimaryKeys());
+
+      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("records were processed from the file").hasStatus(Status.INFO);
+      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id values between 1 and 2").hasStatus(Status.INFO);
 
       ////////////////////////////////////
       // query for the inserted records //
@@ -201,6 +220,86 @@ class BulkInsertFullProcessTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSummaryLinePrimaryKeys() throws Exception
+   {
+      assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty();
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
+         .withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(PersonWarnOrErrorCustomizer.class));
+
+      /////////////////////////////////////////////////////////
+      // start the process - expect to go to the upload step //
+      /////////////////////////////////////////////////////////
+      RunProcessInput  runProcessInput  = new RunProcessInput();
+      RunProcessOutput runProcessOutput = startProcess(runProcessInput);
+      String           processUUID      = runProcessOutput.getProcessUUID();
+
+      continueProcessPostUpload(runProcessInput, processUUID, simulateFileUploadForWarningCase());
+      continueProcessPostFileMapping(runProcessInput);
+      continueProcessPostValueMapping(runProcessInput);
+      runProcessOutput = continueProcessPostReviewScreen(runProcessInput);
+
+      ProcessSummaryLineInterface okLine = ProcessSummaryAssert.assertThat(runProcessOutput)
+         .hasLineWithMessageContaining("Person Memory record was inserted")
+         .hasStatus(Status.OK)
+         .hasCount(1)
+         .getLine();
+      assertEquals(List.of(1), ((ProcessSummaryLine) okLine).getPrimaryKeys());
+
+      ProcessSummaryLineInterface warnTornadoLine = ProcessSummaryAssert.assertThat(runProcessOutput)
+         .hasLineWithMessageContaining("records were inserted, but had a warning: Tornado warning")
+         .hasStatus(Status.WARNING)
+         .hasCount(2)
+         .getLine();
+      assertEquals(List.of(2, 3), ((ProcessSummaryLine) warnTornadoLine).getPrimaryKeys());
+
+      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("record was inserted, but had a warning: Hurricane warning").hasStatus(Status.WARNING).hasCount(1);
+      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("records were processed from the file").hasStatus(Status.INFO).hasCount(4);
+      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id values between 1 and 4").hasStatus(Status.INFO);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSummaryLineErrors() throws Exception
+   {
+      assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty();
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
+         .withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(PersonWarnOrErrorCustomizer.class));
+
+      /////////////////////////////////////////////////////////
+      // start the process - expect to go to the upload step //
+      /////////////////////////////////////////////////////////
+      RunProcessInput  runProcessInput  = new RunProcessInput();
+      RunProcessOutput runProcessOutput = startProcess(runProcessInput);
+      String           processUUID      = runProcessOutput.getProcessUUID();
+
+      continueProcessPostUpload(runProcessInput, processUUID, simulateFileUploadForErrorCase());
+      continueProcessPostFileMapping(runProcessInput);
+      continueProcessPostValueMapping(runProcessInput);
+      runProcessOutput = continueProcessPostReviewScreen(runProcessInput);
+
+      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Person Memory record was inserted.").hasStatus(Status.OK).hasCount(1);
+
+      ProcessSummaryAssert.assertThat(runProcessOutput)
+         .hasLineWithMessageContaining("plane")
+         .hasStatus(Status.ERROR)
+         .hasCount(1);
+
+      ProcessSummaryAssert.assertThat(runProcessOutput)
+         .hasLineWithMessageContaining("purifier")
+         .hasStatus(Status.ERROR)
+         .hasCount(1);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -301,6 +400,47 @@ class BulkInsertFullProcessTest extends BaseTest
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static StorageInput simulateFileUploadForWarningCase() throws Exception
+   {
+      String       storageReference = UUID.randomUUID() + ".csv";
+      StorageInput storageInput     = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference);
+      try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput))
+      {
+         outputStream.write((getPersonCsvHeaderUsingLabels() + """
+            "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com","Missouri",42
+            "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Tornado warning","Doe","1980-01-01","john@doe.com","Missouri",42
+            "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Tornado warning","Doey","1980-01-01","john@doe.com","Missouri",42
+            "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Hurricane warning","Doe","1980-01-01","john@doe.com","Missouri",42
+            """).getBytes());
+      }
+      return storageInput;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static StorageInput simulateFileUploadForErrorCase() throws Exception
+   {
+      String       storageReference = UUID.randomUUID() + ".csv";
+      StorageInput storageInput     = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference);
+      try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput))
+      {
+         outputStream.write((getPersonCsvHeaderUsingLabels() + """
+            "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com","Missouri",42
+            "0","2021-10-26 14:39:37","2021-10-26 14:39:37","not-pre-Error plane","Doe","1980-01-01","john@doe.com","Missouri",42
+            "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Error purifier","Doe","1980-01-01","john@doe.com","Missouri",42
+            """).getBytes());
+      }
+      return storageInput;
+   }
+
+
+
    /***************************************************************************
     **
     ***************************************************************************/
@@ -331,4 +471,47 @@ class BulkInsertFullProcessTest extends BaseTest
       )));
    }
 
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static class PersonWarnOrErrorCustomizer implements TableCustomizerInterface
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public AbstractPreInsertCustomizer.WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview)
+      {
+         return AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS;
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
+      {
+         for(QRecord record : records)
+         {
+            if(record.getValueString("firstName").toLowerCase().contains("warn"))
+            {
+               record.addWarning(new QWarningMessage(record.getValueString("firstName")));
+            }
+            else if(record.getValueString("firstName").toLowerCase().contains("error"))
+            {
+               if(isPreview && record.getValueString("firstName").toLowerCase().contains("not-pre-error"))
+               {
+                  continue;
+               }
+
+               record.addError(new BadInputStatusMessage(record.getValueString("firstName")));
+            }
+         }
+         return records;
+      }
+   }
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java

@@ -22,8 +22,15 @@
 package com.kingsrook.qqq.backend.core.processes.utils;
 
 
+import java.util.ArrayList;
+import java.util.ConcurrentModificationException;
 import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -32,6 +39,7 @@ import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Fail.fail;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
@@ -195,4 +203,54 @@ class RecordLookupHelperTest extends BaseTest
       assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
    }
 
+
+
+   /*******************************************************************************
+    ** run a lot of threads (eg, 100), each trying to do lots of work in a
+    ** shared recordLookupHelper.  w/o the flag to use sync'ed collections, this
+    ** (usually?) fails with a ConcurrentModificationException - but with the sync'ed
+    ** collections, is safe.
+    *******************************************************************************/
+   @Test
+   void testConcurrentModification() throws InterruptedException, ExecutionException
+   {
+      ExecutorService    executorService    = Executors.newFixedThreadPool(100);
+      RecordLookupHelper recordLookupHelper = new RecordLookupHelper(true);
+
+      CapturedContext capture = QContext.capture();
+
+      List<Future<?>> futures = new ArrayList<>();
+      for(int i = 0; i < 100; i++)
+      {
+         int finalI = i;
+         Future<?> future = executorService.submit(() ->
+         {
+            QContext.init(capture);
+            for(int j = 0; j < 25000; j++)
+            {
+               try
+               {
+                  recordLookupHelper.getRecordByKey(String.valueOf(j), "id", j);
+               }
+               catch(ConcurrentModificationException cme)
+               {
+                  fail("CME!", cme);
+               }
+               catch(Exception e)
+               {
+                  //////////////
+                  // expected //
+                  //////////////
+               }
+            }
+         });
+         futures.add(future);
+      }
+
+      for(Future<?> future : futures)
+      {
+         future.get();
+      }
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java

@@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.state;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -103,17 +105,17 @@ public class InMemoryStateProviderTest extends BaseTest
       /////////////////////////////////////////////////////////////
       // Add an entry that is 3 hours old, should not be cleaned //
       /////////////////////////////////////////////////////////////
-      UUIDAndTypeStateKey   newKey           = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(3, ChronoUnit.HOURS));
-      String  newUUID    = UUID.randomUUID().toString();
-      QRecord newQRecord = new QRecord().withValue("uuid", newUUID);
+      UUIDAndTypeStateKey newKey     = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(3, ChronoUnit.HOURS));
+      String              newUUID    = UUID.randomUUID().toString();
+      QRecord             newQRecord = new QRecord().withValue("uuid", newUUID);
       stateProvider.put(newKey, newQRecord);
 
       ////////////////////////////////////////////////////////////
       // Add an entry that is 5 hours old, it should be cleaned //
       ////////////////////////////////////////////////////////////
-      UUIDAndTypeStateKey   oldKey           = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(5, ChronoUnit.HOURS));
-      String  oldUUID    = UUID.randomUUID().toString();
-      QRecord oldQRecord = new QRecord().withValue("uuid", oldUUID);
+      UUIDAndTypeStateKey oldKey     = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(5, ChronoUnit.HOURS));
+      String              oldUUID    = UUID.randomUUID().toString();
+      QRecord             oldQRecord = new QRecord().withValue("uuid", oldUUID);
       stateProvider.put(oldKey, oldQRecord);
 
       ///////////////////
@@ -125,7 +127,33 @@ public class InMemoryStateProviderTest extends BaseTest
       Assertions.assertEquals(newUUID, qRecordFromState.getValueString("uuid"), "Should read value from state persistence");
 
       Assertions.assertTrue(stateProvider.get(QRecord.class, oldKey).isEmpty(), "Key not found in state should return empty");
+   }
 
+
+
+   /*******************************************************************************
+    ** originally written with N=100000, but showed the error as small as 1000.
+    *******************************************************************************/
+   @Test
+   void testDemonstrateConcurrentModificationIfNonSynchronizedMap()
+   {
+      int N = 1000;
+      InMemoryStateProvider stateProvider = InMemoryStateProvider.getInstance();
+
+      ExecutorService executorService = Executors.newFixedThreadPool(10);
+      executorService.submit(() ->
+      {
+         for(int i = 0; i < N; i++)
+         {
+            UUIDAndTypeStateKey oldKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(5, ChronoUnit.HOURS));
+            stateProvider.put(oldKey, UUID.randomUUID());
+         }
+      });
+
+      for(int i = 0; i < N; i++)
+      {
+         stateProvider.clean(Instant.now().minus(4, ChronoUnit.HOURS));
+      }
    }
 
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java

@@ -350,4 +350,22 @@ class StringUtilsTest extends BaseTest
       assertEquals("test ((2)) (101)", StringUtils.appendIncrementingSuffix("test ((2)) (100)"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSafeEqualsIgnoreCase()
+   {
+      assertTrue(StringUtils.safeEqualsIgnoreCase(null, null));
+      assertFalse(StringUtils.safeEqualsIgnoreCase("a", null));
+      assertFalse(StringUtils.safeEqualsIgnoreCase(null, "a"));
+      assertTrue(StringUtils.safeEqualsIgnoreCase("a", "a"));
+      assertTrue(StringUtils.safeEqualsIgnoreCase("A", "a"));
+      assertFalse(StringUtils.safeEqualsIgnoreCase("a", "b"));
+      assertTrue(StringUtils.safeEqualsIgnoreCase("timothy d. chamberlain", "TIMOThy d. chaMberlain"));
+      assertTrue(StringUtils.safeEqualsIgnoreCase("timothy d. chamberlain", "timothy d. chamberlain"));
+   }
+
 }

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiFieldUtils.java

@@ -0,0 +1,106 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.api.actions;
+
+
+import com.kingsrook.qqq.api.model.APIVersionRange;
+import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData;
+import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaDataContainer;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import org.apache.commons.lang.BooleanUtils;
+
+
+/*******************************************************************************
+ ** utility methods for working with fields
+ **
+ *******************************************************************************/
+public class ApiFieldUtils
+{
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static boolean isIncluded(String apiName, QFieldMetaData field)
+   {
+      ApiFieldMetaData apiFieldMetaData = getApiFieldMetaData(apiName, field);
+      if(apiFieldMetaData != null && BooleanUtils.isTrue(apiFieldMetaData.getIsExcluded()))
+      {
+         return (false);
+      }
+
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static APIVersionRange getApiVersionRangeForRemovedField(String apiName, QFieldMetaData field)
+   {
+      ApiFieldMetaData apiFieldMetaData = getApiFieldMetaData(apiName, field);
+      if(apiFieldMetaData != null && apiFieldMetaData.getInitialVersion() != null)
+      {
+         if(StringUtils.hasContent(apiFieldMetaData.getFinalVersion()))
+         {
+            return (APIVersionRange.betweenAndIncluding(apiFieldMetaData.getInitialVersion(), apiFieldMetaData.getFinalVersion()));
+         }
+         else
+         {
+            throw (new IllegalStateException("RemovedApiFieldMetaData for field [" + field.getName() + "] did not specify a finalVersion."));
+         }
+      }
+      else
+      {
+         throw (new IllegalStateException("RemovedApiFieldMetaData for field [" + field.getName() + "] did not specify an initialVersion."));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static APIVersionRange getApiVersionRange(String apiName, QFieldMetaData field)
+   {
+      ApiFieldMetaData apiFieldMetaData = getApiFieldMetaData(apiName, field);
+      if(apiFieldMetaData != null && apiFieldMetaData.getInitialVersion() != null)
+      {
+         return (APIVersionRange.afterAndIncluding(apiFieldMetaData.getInitialVersion()));
+      }
+
+      return (APIVersionRange.none());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static ApiFieldMetaData getApiFieldMetaData(String apiName, QFieldMetaData field)
+   {
+      return ObjectUtils.tryAndRequireNonNullElse(() -> ApiFieldMetaDataContainer.of(field).getApiFieldMetaData(apiName), new ApiFieldMetaData());
+   }
+}

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java

@@ -111,7 +111,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
 
    public static final String QUERY_DESCRIPTION = """
       Execute a query on this table, using query criteria as specified in query string parameters.
-                     
+      
       * Pagination is managed via the `pageNo` & `pageSize` query string parameters.  pageNo starts at 1.  pageSize defaults to 50.
       * By default, the response includes the total count of records that match the query criteria.  The count can be omitted by specifying `includeCount=false`
       * By default, results are sorted by the table's primary key, descending.  This can be changed by specifying the `orderBy` query string parameter, following SQL ORDER BY syntax (e.g., `fieldName1 ASC, fieldName2 DESC`)
@@ -139,26 +139,26 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       * The request body should not include a value for the table's primary key.  Rather, a value will be generated and returned in a successful response's body.
       * Any unrecognized field names in the body will cause a 400 error.
       * Any read-only (non-editable) fields provided in the body will be silently ignored.
-                     
+      
       Upon success, a status code of 201 (`Created`) is returned, and the generated value for the primary key will be returned in the response body object.
       """;
 
    public static final String UPDATE_DESCRIPTION = """
       Update one record in this table, by specifying its primary key as a path parameter, and by supplying values to be updated in the request body.
-                     
+      
       * Only the fields provided in the request body will be updated.
       * To remove a value from a field, supply the key for the field, with a null value.
       * The request body does not need to contain all fields from the table.  Rather, only the fields to be updated should be supplied.
       * Any unrecognized field names in the body will cause a 400 error.
       * Any read-only (non-editable) fields provided in the body will be silently ignored.
       * Note that if the request body includes the primary key, it will be ignored.  Only the primary key value path parameter will be used.
-                     
+      
       Upon success, a status code of 204 (`No Content`) is returned, with no response body.
       """;
 
    public static final String DELETE_DESCRIPTION = """
       Delete one record from this table, by specifying its primary key as a path parameter.
-                     
+      
       Upon success, a status code of 204 (`No Content`) is returned, with no response body.
       """;
 
@@ -167,7 +167,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       * The objects in the request body should not include a value for the table's primary key.  Rather, a value will be generated and returned in a successful response's body
       * Any unrecognized field names in the body will cause a 400 error.
       * Any read-only (non-editable) fields provided in the body will be silently ignored.
-                     
+      
       An HTTP 207 (`Multi-Status`) code is generally returned, with an array of objects giving the individual sub-status codes for each record in the request body.
       * The 1st record in the request will have its response in the 1st object in the response, and so-forth.
       * For sub-status codes of 201 (`Created`), and the generated value for the primary key will be returned in the response body object.
@@ -180,7 +180,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       * The request body does not need to contain all fields from the table.  Rather, only the fields to be updated should be supplied.
       * Any unrecognized field names in the body will cause a 400 error.
       * Any read-only (non-editable) fields provided in the body will be silently ignored.
-                    
+      
       An HTTP 207 (`Multi-Status`) code is generally returned, with an array of objects giving the individual sub-status codes for each record in the request body.
       * The 1st record in the request will have its response in the 1st object in the response, and so-forth.
       * Each input object's primary key will also be included in the corresponding response object.
@@ -188,7 +188,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
 
    public static final String BULK_DELETE_DESCRIPTION = """
       Delete one or more records from this table, by supplying an array of primary key values in the request body.
-                     
+      
       An HTTP 207 (`Multi-Status`) code is generally returned, with an array of objects giving the individual sub-status codes for each record in the request body.
       * The 1st primary key in the request will have its response in the 1st object in the response, and so-forth.
       * Each input primary key will also be included in the corresponding response object.
@@ -726,7 +726,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
             ApiProcessMetaData apiProcessMetaData = pair.getA();
             QProcessMetaData   processMetaData    = pair.getB();
 
-            addProcessEndpoints(qInstance, apiInstanceMetaData, basePath, openAPI, tableProcessesTag, apiProcessMetaData, processMetaData);
+            addProcessEndpoints(qInstance, apiInstanceMetaData, basePath, openAPI, tableProcessesTag, apiProcessMetaData, processMetaData, apiVersion);
 
             usedProcessNames.add(processMetaData.getName());
          }
@@ -761,7 +761,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
                .withName(tag)
                .withDescription(tag));
 
-            addProcessEndpoints(qInstance, apiInstanceMetaData, basePath, openAPI, tag, apiProcessMetaData, processMetaData);
+            addProcessEndpoints(qInstance, apiInstanceMetaData, basePath, openAPI, tag, apiProcessMetaData, processMetaData, apiVersion);
 
             usedProcessNames.add(processMetaData.getName());
          }
@@ -807,14 +807,14 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void addProcessEndpoints(QInstance qInstance, ApiInstanceMetaData apiInstanceMetaData, String basePath, OpenAPI openAPI, String tag, ApiProcessMetaData apiProcessMetaData, QProcessMetaData processMetaData)
+   private void addProcessEndpoints(QInstance qInstance, ApiInstanceMetaData apiInstanceMetaData, String basePath, OpenAPI openAPI, String tag, ApiProcessMetaData apiProcessMetaData, QProcessMetaData processMetaData, APIVersion apiVersion)
    {
       String processApiPath = ApiProcessUtils.getProcessApiPath(qInstance, processMetaData, apiProcessMetaData, apiInstanceMetaData);
 
       ///////////////////////////
       // do the process itself //
       ///////////////////////////
-      Path path = generateProcessSpecPathObject(apiInstanceMetaData, apiProcessMetaData, processMetaData, ListBuilder.of(tag));
+      Path path = generateProcessSpecPathObject(apiInstanceMetaData, apiProcessMetaData, processMetaData, ListBuilder.of(tag), apiVersion);
       openAPI.getPaths().put(basePath + processApiPath, path);
 
       ///////////////////////////////////////////////////////////////////////
@@ -872,7 +872,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
    /*******************************************************************************
     **
     *******************************************************************************/
-   private Path generateProcessSpecPathObject(ApiInstanceMetaData apiInstanceMetaData, ApiProcessMetaData apiProcessMetaData, QProcessMetaData processMetaData, List<String> tags)
+   private Path generateProcessSpecPathObject(ApiInstanceMetaData apiInstanceMetaData, ApiProcessMetaData apiProcessMetaData, QProcessMetaData processMetaData, List<String> tags, APIVersion apiVersion)
    {
       String description = apiProcessMetaData.getDescription();
       if(!StringUtils.hasContent(description))
@@ -927,7 +927,10 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
 
             for(QFieldMetaData field : CollectionUtils.nonNullList(queryStringParams.getFields()))
             {
-               parameters.add(processFieldToParameter(apiInstanceMetaData, field).withIn("query"));
+               if(ApiFieldUtils.isIncluded(apiName, field) && ApiFieldUtils.getApiVersionRange(apiName, field).includes(apiVersion))
+               {
+                  parameters.add(processFieldToParameter(apiInstanceMetaData, field).withIn("query"));
+               }
             }
          }
 

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GetTableApiFieldsAction.java

@@ -31,11 +31,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import com.kingsrook.qqq.api.model.APIVersion;
-import com.kingsrook.qqq.api.model.APIVersionRange;
 import com.kingsrook.qqq.api.model.actions.GetTableApiFieldsInput;
 import com.kingsrook.qqq.api.model.actions.GetTableApiFieldsOutput;
 import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData;
-import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaDataContainer;
 import com.kingsrook.qqq.api.model.metadata.tables.ApiTableMetaData;
 import com.kingsrook.qqq.api.model.metadata.tables.ApiTableMetaDataContainer;
 import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
@@ -45,8 +43,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
-import org.apache.commons.lang.BooleanUtils;
 
 
 /*******************************************************************************
@@ -160,7 +156,7 @@ public class GetTableApiFieldsAction extends AbstractQActionFunction<GetTableApi
       fieldList.sort(Comparator.comparing(QFieldMetaData::getLabel));
       for(QFieldMetaData field : fieldList)
       {
-         if(!isExcluded(input.getApiName(), field) && getApiVersionRange(input.getApiName(), field).includes(version))
+         if(ApiFieldUtils.isIncluded(input.getApiName(), field) && ApiFieldUtils.getApiVersionRange(input.getApiName(), field).includes(version))
          {
             fields.add(field);
          }
@@ -171,7 +167,7 @@ public class GetTableApiFieldsAction extends AbstractQActionFunction<GetTableApi
       //////////////////////////////////////////////////////////////////////////////////////////////////
       for(QFieldMetaData field : CollectionUtils.nonNullList(getRemovedApiFields(input.getApiName(), table)))
       {
-         if(!isExcluded(input.getApiName(), field) && getApiVersionRangeForRemovedField(input.getApiName(), field).includes(version))
+         if(ApiFieldUtils.isIncluded(input.getApiName(), field) && ApiFieldUtils.getApiVersionRangeForRemovedField(input.getApiName(), field).includes(version))
          {
             fields.add(field);
          }
@@ -184,73 +180,6 @@ public class GetTableApiFieldsAction extends AbstractQActionFunction<GetTableApi
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private boolean isExcluded(String apiName, QFieldMetaData field)
-   {
-      ApiFieldMetaData apiFieldMetaData = getApiFieldMetaData(apiName, field);
-      if(apiFieldMetaData != null && BooleanUtils.isTrue(apiFieldMetaData.getIsExcluded()))
-      {
-         return (true);
-      }
-
-      return (false);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private APIVersionRange getApiVersionRangeForRemovedField(String apiName, QFieldMetaData field)
-   {
-      ApiFieldMetaData apiFieldMetaData = getApiFieldMetaData(apiName, field);
-      if(apiFieldMetaData != null && apiFieldMetaData.getInitialVersion() != null)
-      {
-         if(StringUtils.hasContent(apiFieldMetaData.getFinalVersion()))
-         {
-            return (APIVersionRange.betweenAndIncluding(apiFieldMetaData.getInitialVersion(), apiFieldMetaData.getFinalVersion()));
-         }
-         else
-         {
-            throw (new IllegalStateException("RemovedApiFieldMetaData for field [" + field.getName() + "] did not specify a finalVersion."));
-         }
-      }
-      else
-      {
-         throw (new IllegalStateException("RemovedApiFieldMetaData for field [" + field.getName() + "] did not specify an initialVersion."));
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private APIVersionRange getApiVersionRange(String apiName, QFieldMetaData field)
-   {
-      ApiFieldMetaData apiFieldMetaData = getApiFieldMetaData(apiName, field);
-      if(apiFieldMetaData != null && apiFieldMetaData.getInitialVersion() != null)
-      {
-         return (APIVersionRange.afterAndIncluding(apiFieldMetaData.getInitialVersion()));
-      }
-
-      return (APIVersionRange.none());
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static ApiFieldMetaData getApiFieldMetaData(String apiName, QFieldMetaData field)
-   {
-      return ObjectUtils.tryAndRequireNonNullElse(() -> ApiFieldMetaDataContainer.of(field).getApiFieldMetaData(apiName), new ApiFieldMetaData());
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/utils/ProcessSpecUtilsV1.java

@@ -163,7 +163,6 @@ public class ProcessSpecUtilsV1
       JSONObject outputJsonObject = convertResponseToJSONObject(response);
 
       String json = outputJsonObject.toString(3);
-      System.out.println(json);
       context.result(json);
    }
 
@@ -308,8 +307,8 @@ public class ProcessSpecUtilsV1
    private static void archiveUploadedFile(String processName, QUploadedFile qUploadedFile)
    {
       String fileName = QValueFormatter.formatDate(LocalDate.now())
-                        + File.separator + processName
-                        + File.separator + qUploadedFile.getFilename();
+         + File.separator + processName
+         + File.separator + qUploadedFile.getFilename();
 
       InsertInput insertInput = new InsertInput();
       insertInput.setTableName(QJavalinImplementation.getJavalinMetaData().getUploadedFileArchiveTableName());