CE-1955 Add mapping and validation of possible-values; refactor error classes some for rollup possible value errors

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java

@@ -56,8 +56,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.AbstractBulkLoadRollableValueError;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils;
-import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
@@ -210,10 +210,10 @@ public class BulkInsertTransformStep extends AbstractTransformStep
          {
             for(QErrorMessage error : record.getErrors())
             {
-               if(error instanceof BulkLoadValueTypeError blvte)
+               if(error instanceof AbstractBulkLoadRollableValueError rollableValueError)
                {
-                  processSummaryWarningsAndErrorsRollup.addError(blvte.getMessageToUseAsProcessSummaryRollupKey(), null);
+                  processSummaryWarningsAndErrorsRollup.addError(rollableValueError.getMessageToUseAsProcessSummaryRollupKey(), null);
-                  addToErrorToExampleRowValueMap(blvte, record);
+                  addToErrorToExampleRowValueMap(rollableValueError, record);
                }
                else
                {
@@ -348,14 +348,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep
    /***************************************************************************
     **
     ***************************************************************************/
-   private void addToErrorToExampleRowValueMap(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record)
+   private void addToErrorToExampleRowValueMap(AbstractBulkLoadRollableValueError bulkLoadRollableValueError, QRecord record)
    {
-      String         message   = bulkLoadValueTypeError.getMessageToUseAsProcessSummaryRollupKey();
+      String         message   = bulkLoadRollableValueError.getMessageToUseAsProcessSummaryRollupKey();
       List<RowValue> rowValues = errorToExampleRowValueMap.computeIfAbsent(message, k -> new ArrayList<>());
 
       if(rowValues.size() < EXAMPLE_ROW_LIMIT)
       {
-         rowValues.add(new RowValue(bulkLoadValueTypeError, record));
+         rowValues.add(new RowValue(bulkLoadRollableValueError, record));
       }
    }
 
@@ -550,9 +550,9 @@ public class BulkInsertTransformStep extends AbstractTransformStep
       /***************************************************************************
        **
        ***************************************************************************/
-      public RowValue(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record)
+      public RowValue(AbstractBulkLoadRollableValueError bulkLoadRollableValueError, QRecord record)
       {
-         this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadValueTypeError.getValue()));
+         this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadRollableValueError.getValue()));
       }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java

@@ -0,0 +1,55 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
+
+
+import java.io.Serializable;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public abstract class AbstractBulkLoadRollableValueError extends BadInputStatusMessage
+{
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public AbstractBulkLoadRollableValueError(String message)
+   {
+      super(message);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public abstract String getMessageToUseAsProcessSummaryRollupKey();
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public abstract Serializable getValue();
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java

@@ -0,0 +1,72 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
+
+
+import java.io.Serializable;
+
+
+/*******************************************************************************
+ ** Specialized error for records, for bulk-load use-cases, where we want to
+ ** report back info to the user about the field & value.
+ *******************************************************************************/
+public class BulkLoadPossibleValueError extends AbstractBulkLoadRollableValueError
+{
+   private final String       fieldLabel;
+   private final Serializable value;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public BulkLoadPossibleValueError(String fieldName, Serializable value, String fieldLabel)
+   {
+      super("Value [" + value + "] for field [" + fieldLabel + "] is not a valid option");
+      this.value = value;
+      this.fieldLabel = fieldLabel;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public String getMessageToUseAsProcessSummaryRollupKey()
+   {
+      return ("Unrecognized value for field [" + fieldLabel + "]");
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for value
+    **
+    *******************************************************************************/
+   @Override
+   public Serializable getValue()
+   {
+      return value;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java

@@ -0,0 +1,238 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
+import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ListingHash;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class BulkLoadValueMapper
+{
+   private static final QLogger LOG = QLogger.getLogger(BulkLoadValueMapper.class);
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, QTableMetaData table) throws QException
+   {
+      valueMapping(records, mapping, table, null);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException
+   {
+      if(CollectionUtils.nullSafeIsEmpty(records))
+      {
+         return;
+      }
+
+      String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : "";
+      String tableLabelPrefix               = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : "";
+
+      Map<String, ListingHash<String, QRecord>> possibleValueToRecordMap = new HashMap<>();
+
+      Map<String, Map<String, Serializable>> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain);
+      for(QRecord record : records)
+      {
+         for(Map.Entry<String, Serializable> valueEntry : record.getValues().entrySet())
+         {
+            QFieldMetaData field = table.getField(valueEntry.getKey());
+            Serializable   value = valueEntry.getValue();
+
+            ///////////////////
+            // value mappin' //
+            ///////////////////
+            if(mappingForTable.containsKey(field.getName()) && value != null)
+            {
+               Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value));
+               if(mappedValue != null)
+               {
+                  value = mappedValue;
+               }
+            }
+
+            /////////////////////
+            // type convertin' //
+            /////////////////////
+            if(value != null && !"".equals(value))
+            {
+               if(StringUtils.hasContent(field.getPossibleValueSourceName()))
+               {
+                  ListingHash<String, QRecord> fieldPossibleValueToRecordMap = possibleValueToRecordMap.computeIfAbsent(field.getName(), k -> new ListingHash<>());
+                  fieldPossibleValueToRecordMap.add(ValueUtils.getValueAsString(value), record);
+               }
+               else
+               {
+                  QFieldType type = field.getType();
+                  try
+                  {
+                     value = ValueUtils.getValueAsFieldType(type, value);
+                  }
+                  catch(Exception e)
+                  {
+                     record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel()));
+                  }
+               }
+            }
+
+            record.setValue(field.getName(), value);
+         }
+
+         //////////////////////////////////////
+         // recursively process associations //
+         //////////////////////////////////////
+         for(Map.Entry<String, List<QRecord>> entry : record.getAssociatedRecords().entrySet())
+         {
+            String                associationName = entry.getKey();
+            Optional<Association> association     = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst();
+            if(association.isPresent())
+            {
+               QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
+               valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName);
+            }
+            else
+            {
+               throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]");
+            }
+         }
+      }
+
+      //////////////////////////////////////////
+      // look up and validate possible values //
+      //////////////////////////////////////////
+      for(Map.Entry<String, ListingHash<String, QRecord>> entry : possibleValueToRecordMap.entrySet())
+      {
+         String                       fieldName                     = entry.getKey();
+         QFieldMetaData               field                         = table.getField(fieldName);
+         ListingHash<String, QRecord> fieldPossibleValueToRecordMap = possibleValueToRecordMap.get(fieldName);
+
+         handlePossibleValues(field, fieldPossibleValueToRecordMap, associationNamePrefixForFields, tableLabelPrefix);
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static void handlePossibleValues(QFieldMetaData field, ListingHash<String, QRecord> fieldPossibleValueToRecordMap, String associationNamePrefixForFields, String tableLabelPrefix) throws QException
+   {
+      Set<String>                    values         = fieldPossibleValueToRecordMap.keySet();
+      Map<String, QPossibleValue<?>> valuesFound    = new HashMap<>();
+      Set<String>                    valuesNotFound = new HashSet<>(values);
+
+      ////////////////////////////////////////////////////////
+      // do a search, trying to use all given values as ids //
+      ////////////////////////////////////////////////////////
+      SearchPossibleValueSourceInput searchPossibleValueSourceInput = new SearchPossibleValueSourceInput();
+      searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName());
+      ArrayList<Serializable> idList = new ArrayList<>(values);
+      searchPossibleValueSourceInput.setIdList(idList);
+      searchPossibleValueSourceInput.setLimit(values.size());
+      LOG.debug("Searching possible value source by ids during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfIds", idList.size()), logPair("firstId", () -> idList.get(0)));
+      SearchPossibleValueSourceOutput searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput);
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      // for each possible value found, remove it from the set of ones not-found, and store it as a hit //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      for(QPossibleValue<?> possibleValue : searchPossibleValueSourceOutput.getResults())
+      {
+         String valueAsString = ValueUtils.getValueAsString(possibleValue.getId());
+         valuesFound.put(valueAsString, possibleValue);
+         valuesNotFound.remove(valueAsString);
+      }
+
+      ///////////////////////////////////////////////////////////////////////////
+      // if there are any that weren't found, try to look them up now by label //
+      ///////////////////////////////////////////////////////////////////////////
+      if(!valuesNotFound.isEmpty())
+      {
+         searchPossibleValueSourceInput = new SearchPossibleValueSourceInput();
+         searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName());
+         ArrayList<String> labelList = new ArrayList<>(valuesNotFound);
+         searchPossibleValueSourceInput.setLabelList(labelList);
+         searchPossibleValueSourceInput.setLimit(valuesNotFound.size());
+
+         LOG.debug("Searching possible value source by labels during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfLabels", labelList.size()), logPair("firstLabel", () -> labelList.get(0)));
+         searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput);
+         for(QPossibleValue<?> possibleValue : searchPossibleValueSourceOutput.getResults())
+         {
+            valuesFound.put(possibleValue.getLabel(), possibleValue);
+            valuesNotFound.remove(possibleValue.getLabel());
+         }
+      }
+
+      ////////////////////////////////////////////////////////////////////////////////
+      // for each record now, either set a usable value (e.g., a PV.id) or an error //
+      ////////////////////////////////////////////////////////////////////////////////
+      for(Map.Entry<String, List<QRecord>> entry : fieldPossibleValueToRecordMap.entrySet())
+      {
+         String value = entry.getKey();
+         for(QRecord record : entry.getValue())
+         {
+            if(valuesFound.containsKey(value))
+            {
+               record.setValue(field.getName(), valuesFound.get(value).getId());
+            }
+            else
+            {
+               record.addError(new BulkLoadPossibleValueError(associationNamePrefixForFields + field.getName(), value, tableLabelPrefix + field.getLabel()));
+            }
+         }
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java

@@ -24,17 +24,15 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.map
 
 import java.io.Serializable;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
-import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
 
 
 /*******************************************************************************
  ** Specialized error for records, for bulk-load use-cases, where we want to
  ** report back info to the user about the field & value.
  *******************************************************************************/
-public class BulkLoadValueTypeError extends BadInputStatusMessage
+public class BulkLoadValueTypeError extends AbstractBulkLoadRollableValueError
 {
    private final String       fieldLabel;
-   private final String       fieldName;
    private final Serializable value;
    private final QFieldType   type;
 
@@ -47,7 +45,6 @@ public class BulkLoadValueTypeError extends BadInputStatusMessage
    public BulkLoadValueTypeError(String fieldName, Serializable value, QFieldType type, String fieldLabel)
    {
       super("Value [" + value + "] for field [" + fieldLabel + "] could not be converted to type [" + type + "]");
-      this.fieldName = fieldName;
       this.value = value;
       this.type = type;
       this.fieldLabel = fieldLabel;
@@ -58,6 +55,7 @@ public class BulkLoadValueTypeError extends BadInputStatusMessage
    /***************************************************************************
     **
     ***************************************************************************/
+   @Override
    public String getMessageToUseAsProcessSummaryRollupKey()
    {
       return ("Cannot convert value for field [" + fieldLabel + "] to type [" + type.getMixedCaseLabel() + "]");
@@ -69,6 +67,7 @@ public class BulkLoadValueTypeError extends BadInputStatusMessage
     ** Getter for value
     **
     *******************************************************************************/
+   @Override
    public Serializable getValue()
    {
       return value;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java

@@ -70,7 +70,7 @@ public class FlatRowsToRecord implements RowsToRecordInterface
          rs.add(record);
       }
 
-      ValueMapper.valueMapping(rs, mapping, table);
+      BulkLoadValueMapper.valueMapping(rs, mapping, table);
 
       return (rs);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java

@@ -138,7 +138,7 @@ public class TallRowsToRecord implements RowsToRecordInterface
          rs.add(record);
       }
 
-      ValueMapper.valueMapping(rs, mapping, table);
+      BulkLoadValueMapper.valueMapping(rs, mapping, table);
 
       return (rs);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java

@@ -1,134 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
-
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
-import com.kingsrook.qqq.backend.core.utils.ValueUtils;
-
-
-/*******************************************************************************
- **
- *******************************************************************************/
-public class ValueMapper
-{
-   private static final QLogger LOG = QLogger.getLogger(ValueMapper.class);
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   public static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, QTableMetaData table) throws QException
-   {
-      valueMapping(records, mapping, table, null);
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException
-   {
-      if(CollectionUtils.nullSafeIsEmpty(records))
-      {
-         return;
-      }
-
-      String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : "";
-      String tableLabelPrefix = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : "";
-
-      Map<String, Map<String, Serializable>> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain);
-      for(QRecord record : records)
-      {
-         for(Map.Entry<String, Serializable> valueEntry : record.getValues().entrySet())
-         {
-            QFieldMetaData field = table.getField(valueEntry.getKey());
-            Serializable   value = valueEntry.getValue();
-
-            ///////////////////
-            // value mappin' //
-            ///////////////////
-            if(mappingForTable.containsKey(field.getName()) && value != null)
-            {
-               Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value));
-               if(mappedValue != null)
-               {
-                  value = mappedValue;
-               }
-            }
-
-            /////////////////////
-            // type convertin' //
-            /////////////////////
-            if(value != null)
-            {
-               QFieldType type = field.getType();
-               try
-               {
-                  value = ValueUtils.getValueAsFieldType(type, value);
-               }
-               catch(Exception e)
-               {
-                  record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel()));
-               }
-            }
-
-            record.setValue(field.getName(), value);
-         }
-
-         //////////////////////////////////////
-         // recursively process associations //
-         //////////////////////////////////////
-         for(Map.Entry<String, List<QRecord>> entry : record.getAssociatedRecords().entrySet())
-         {
-            String                associationName = entry.getKey();
-            Optional<Association> association     = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst();
-            if(association.isPresent())
-            {
-               QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
-               valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName);
-            }
-            else
-            {
-               throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]");
-            }
-         }
-      }
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java

@@ -75,7 +75,7 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem
          rs.add(record);
       }
 
-      ValueMapper.valueMapping(rs, mapping, table);
+      BulkLoadValueMapper.valueMapping(rs, mapping, table);
 
       return (rs);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java

@@ -86,7 +86,7 @@ public class WideRowsToRecordWithSpreadMapping implements RowsToRecordInterface
          rs.add(record);
       }
 
-      ValueMapper.valueMapping(rs, mapping, table);
+      BulkLoadValueMapper.valueMapping(rs, mapping, table);
 
       return (rs);
    }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java

@@ -41,7 +41,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 /*******************************************************************************
  ** Unit test for ValueMapper 
  *******************************************************************************/
-class ValueMapperTest extends BaseTest
+class BulkLoadValueMapperTest extends BaseTest
 {
 
    /*******************************************************************************
@@ -98,7 +98,7 @@ class ValueMapperTest extends BaseTest
          );
       JSONObject expectedJson = recordToJson(expectedRecord);
 
-      ValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER));
+      BulkLoadValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER));
       JSONObject actualJson = recordToJson(inputRecord);
 
       System.out.println("Before");

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java

@@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mode
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
@@ -102,6 +103,8 @@ class FlatRowsToRecordTest extends BaseTest
       assertEquals("Row 5", records.get(0).getBackendDetail("rowNos"));
    }
 
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -205,6 +208,48 @@ class FlatRowsToRecordTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testPossibleValueMappings() throws QException
+   {
+      TestFileToRows fileToRows = new TestFileToRows(List.of(
+         new Serializable[] { "id", "firstName", "Last Name", "Home State" },
+         new Serializable[] { 1, "Homer", "Simpson", 1 },
+         new Serializable[] { 2, "Marge", "Simpson", "MO" },
+         new Serializable[] { 3, "Bart", "Simpson", null },
+         new Serializable[] { 4, "Ned", "Flanders", "Not a state" },
+         new Serializable[] { 5, "Mr.", "Burns", 5 }
+      ));
+
+      BulkLoadFileRow header = fileToRows.next();
+
+      FlatRowsToRecord rowsToRecord = new FlatRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "firstName", "firstName",
+            "lastName", "Last Name",
+            "homeStateId", "Home State"
+         ))
+         .withTableName(TestUtils.TABLE_NAME_PERSON)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(5, records.size());
+      assertEquals(List.of("Homer", "Marge", "Bart", "Ned", "Mr."), getValues(records, "firstName"));
+      assertEquals(ListBuilder.of(1, 2, null, "Not a state", 5), getValues(records, "homeStateId"));
+
+      assertThat(records.get(0).getErrors()).isNullOrEmpty();
+      assertThat(records.get(1).getErrors()).isNullOrEmpty();
+      assertThat(records.get(2).getErrors()).isNullOrEmpty();
+      assertThat(records.get(3).getErrors()).hasSize(1).element(0).matches(e -> e.getMessage().contains("not a valid option"));
+      assertThat(records.get(4).getErrors()).hasSize(1).element(0).matches(e -> e.getMessage().contains("not a valid option"));
+   }
+
+
+
    /***************************************************************************
     **
     ***************************************************************************/