CE-604 Bulk load improvements - type handling (via csv adapter) and better errors

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java

@@ -36,7 +36,9 @@ import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFiel
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVParser;
 import org.apache.commons.csv.CSVRecord;
@@ -156,14 +158,21 @@ public class CsvToQRecordAdapter
                // now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
                //////////////////////////////////////////////////////////////////////////////////////////////////////////
                QRecord qRecord = new QRecord();
-               for(QFieldMetaData field : table.getFields().values())
+               try
                {
-                  String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
+                  for(QFieldMetaData field : table.getFields().values())
-                  fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
+                  {
-                  qRecord.setValue(field.getName(), csvValues.get(fieldSource));
+                     String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
-               }
+                     fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
+                     setValue(inputWrapper, qRecord, field, csvValues.get(fieldSource));
+                  }
 
-               runRecordCustomizer(recordCustomizer, qRecord);
+                  runRecordCustomizer(recordCustomizer, qRecord);
+               }
+               catch(Exception e)
+               {
+                  qRecord.addError(new BadInputStatusMessage("Error parsing line #" + (recordCount + 1) + ": " + e.getMessage()));
+               }
                addRecord(qRecord);
 
                recordCount++;
@@ -202,13 +211,20 @@ public class CsvToQRecordAdapter
                // now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
                //////////////////////////////////////////////////////////////////////////////////////////////////////////
                QRecord qRecord = new QRecord();
-               for(QFieldMetaData field : table.getFields().values())
+               try
                {
-                  Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName());
+                  for(QFieldMetaData field : table.getFields().values())
-                  qRecord.setValue(field.getName(), csvValues.get(fieldIndex));
+                  {
-               }
+                     Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName());
+                     setValue(inputWrapper, qRecord, field, csvValues.get(fieldIndex));
+                  }
 
-               runRecordCustomizer(recordCustomizer, qRecord);
+                  runRecordCustomizer(recordCustomizer, qRecord);
+               }
+               catch(Exception e)
+               {
+                  qRecord.addError(new BadInputStatusMessage("Error parsing line #" + (recordCount + 1) + ": " + e.getMessage()));
+               }
                addRecord(qRecord);
 
                recordCount++;
@@ -231,6 +247,23 @@ public class CsvToQRecordAdapter
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void setValue(InputWrapper inputWrapper, QRecord qRecord, QFieldMetaData field, String valueString)
+   {
+      if(inputWrapper.doCorrectValueTypes)
+      {
+         qRecord.setValue(field.getName(), ValueUtils.getValueAsFieldType(field.getType(), valueString));
+      }
+      else
+      {
+         qRecord.setValue(field.getName(), valueString);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -341,6 +374,7 @@ public class CsvToQRecordAdapter
       private AbstractQFieldMapping<?> mapping;
       private Consumer<QRecord>        recordCustomizer;
       private Integer                  limit;
+      private boolean                  doCorrectValueTypes = false;
 
       private boolean caseSensitiveHeaders = false;
 
@@ -582,6 +616,40 @@ public class CsvToQRecordAdapter
          return (this);
       }
 
+
+
+      /*******************************************************************************
+       ** Getter for doCorrectValueTypes
+       **
+       *******************************************************************************/
+      public boolean getDoCorrectValueTypes()
+      {
+         return doCorrectValueTypes;
+      }
+
+
+
+      /*******************************************************************************
+       ** Setter for doCorrectValueTypes
+       **
+       *******************************************************************************/
+      public void setDoCorrectValueTypes(boolean doCorrectValueTypes)
+      {
+         this.doCorrectValueTypes = doCorrectValueTypes;
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for doCorrectValueTypes
+       **
+       *******************************************************************************/
+      public InputWrapper withDoCorrectValueTypes(boolean doCorrectValueTypes)
+      {
+         this.doCorrectValueTypes = doCorrectValueTypes;
+         return (this);
+      }
+
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java

@@ -82,6 +82,7 @@ public class BulkInsertExtractStep extends AbstractExtractStep
             .withRecordPipe(getRecordPipe())
             .withLimit(getLimit())
             .withCsv(new String(bytes))
+            .withDoCorrectValueTypes(true)
             .withTable(runBackendStepInput.getInstance().getTable(tableName))
             .withMapping(mapping)
             .withRecordCustomizer((record) ->

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -75,11 +76,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep
 
 
    /*******************************************************************************
-    **
+    ** extension of ProcessSummaryLine for lines where a UniqueKey was violated,
+    ** where we'll collect a sample (or maybe all) of the values that broke the UK.
     *******************************************************************************/
    private static class ProcessSummaryLineWithUKSampleValues extends ProcessSummaryLine
    {
-      private List<String> sampleValues = new ArrayList<>();
+      private Set<String> sampleValues             = new LinkedHashSet<>();
+      private boolean     areThereMoreSampleValues = false;
 
 
 
@@ -116,8 +119,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep
    @Override
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
-      int rowsInThisPage = runBackendStepInput.getRecords().size();
+      int            rowsInThisPage = runBackendStepInput.getRecords().size();
-      QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
+      QTableMetaData table          = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
 
       /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations //
@@ -278,6 +281,10 @@ public class BulkInsertTransformStep extends AbstractTransformStep
                   {
                      processSummaryLineWithUKSampleValues.sampleValues.add(keyValues.get().toString());
                   }
+                  else
+                  {
+                     processSummaryLineWithUKSampleValues.areThereMoreSampleValues = true;
+                  }
                   foundDupe = true;
                   break;
                }
@@ -325,13 +332,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep
          ProcessSummaryLineWithUKSampleValues ukErrorSummary = entry.getValue();
 
          ukErrorSummary
-            .withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values such as: "
+            .withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values"
-               + StringUtils.joinWithCommasAndAnd(ukErrorSummary.sampleValues))
+               + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ")
+               + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues)))
 
-            .withSingularFutureMessage(tableLabel + " record will not be")
+            .withSingularFutureMessage(" record will not be")
-            .withPluralFutureMessage(tableLabel + " records will not be")
+            .withPluralFutureMessage(" records will not be")
-            .withSingularPastMessage(tableLabel + " record was not")
+            .withSingularPastMessage(" record was not")
-            .withPluralPastMessage(tableLabel + " records were not");
+            .withPluralPastMessage(" records were not");
 
          ukErrorSummary.addSelfToListIfAnyCount(rs);
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java

@@ -30,18 +30,20 @@ import java.util.Map;
 import java.util.Objects;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryRecordLink;
 import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.utils.collections.AlphaNumericComparator;
 
 
 /*******************************************************************************
  ** Helper class for process steps that want to roll up error summary and/or
  ** warning summary lines.  e.g., if the process might have a handful of different
- ** error messages.  Will record up to 50 unique errors, then throw the rest int
+ ** error messages.  Will record up to 50 (configurable) unique errors, then throw the rest int
  ** an "other" errors summary.
  *******************************************************************************/
 public class ProcessSummaryWarningsAndErrorsRollup
 {
-   private Map<String, ProcessSummaryLine> errorSummaries   = new HashMap<>();
+   private Map<String, ProcessSummaryLine> errorSummaries = new HashMap<>();
    private Map<String, ProcessSummaryLine> warningSummaries = new HashMap<>();
 
    private ProcessSummaryLine otherErrorsSummary;
@@ -49,6 +51,8 @@ public class ProcessSummaryWarningsAndErrorsRollup
    private ProcessSummaryLine errorTemplate;
    private ProcessSummaryLine warningTemplate;
 
+   private int     uniqueErrorsToShow                         = 50;
+   private boolean doReplaceSingletonCountLinesWithSuffixOnly = true;
 
 
    /*******************************************************************************
@@ -167,7 +171,7 @@ public class ProcessSummaryWarningsAndErrorsRollup
       ProcessSummaryLine processSummaryLine = summaryLineMap.get(message);
       if(processSummaryLine == null)
       {
-         if(summaryLineMap.size() < 50)
+         if(summaryLineMap.size() < uniqueErrorsToShow)
          {
             processSummaryLine = new ProcessSummaryLine(status)
                .withMessageSuffix(message)
@@ -210,17 +214,80 @@ public class ProcessSummaryWarningsAndErrorsRollup
 
 
 
+   /*******************************************************************************
+    ** Wrapper around AlphaNumericComparator for ProcessSummaryLineInterface that
+    ** extracts string messages out.
+    **
+    ** Makes errors from bulk-insert look better when they report, e.g.
+    ** Error parsing line #1: ...
+    ** Error parsing line #2: ...
+    ** Error parsing line #10: ...
+    *******************************************************************************/
+   private static class PSLAlphaNumericComparator implements Comparator<ProcessSummaryLineInterface>
+   {
+      private static AlphaNumericComparator alphaNumericComparator = new AlphaNumericComparator();
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public int compare(ProcessSummaryLineInterface psli1, ProcessSummaryLineInterface psli2)
+      {
+         int messageComp = (alphaNumericComparator.compare(Objects.requireNonNullElse(psli1.getMessage(), ""), Objects.requireNonNullElse(psli2.getMessage(), "")));
+         if(messageComp != 0)
+         {
+            return (messageComp);
+         }
+
+         if(psli1 instanceof ProcessSummaryLine psl1 && psli2 instanceof ProcessSummaryLine psl2)
+         {
+            return (alphaNumericComparator.compare(Objects.requireNonNullElse(psl1.getMessageSuffix(), ""), Objects.requireNonNullElse(psl2.getMessageSuffix(), "")));
+         }
+
+         return (0);
+      }
+   }
+
+
+
    /*******************************************************************************
     ** sort the process summary lines by count desc
     *******************************************************************************/
-   private static void addProcessSummaryLinesFromMap(ArrayList<ProcessSummaryLineInterface> rs, Map<String, ProcessSummaryLine> summaryMap)
+   private void addProcessSummaryLinesFromMap(ArrayList<ProcessSummaryLineInterface> rs, Map<String, ProcessSummaryLine> summaryMap)
    {
       summaryMap.values().stream()
          .sorted(Comparator.comparing((ProcessSummaryLine psl) -> Objects.requireNonNullElse(psl.getCount(), 0)).reversed()
-            .thenComparing((ProcessSummaryLine psl) -> Objects.requireNonNullElse(psl.getMessage(), ""))
+            .thenComparing(new PSLAlphaNumericComparator())
-            .thenComparing((ProcessSummaryLine psl) -> Objects.requireNonNullElse(psl.getMessageSuffix(), ""))
          )
-         .forEach(psl -> psl.addSelfToListIfAnyCount(rs));
+         .map(psl ->
+         {
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // this is to make lines that are like "1 record had an error:  Error parsing line #1:  blah" look better, by //
+            // removing the redundant "1 record..." bit.                                                                  //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(doReplaceSingletonCountLinesWithSuffixOnly)
+            {
+               if(psl.getCount() == 1)
+               {
+                  return (new ProcessSummaryRecordLink().withStatus(Status.ERROR).withLinkPreText(psl.getMessageSuffix()));
+               }
+            }
+
+            return (psl);
+         })
+         .forEach(psli ->
+         {
+            if(psli instanceof ProcessSummaryLine psl)
+            {
+               psl.addSelfToListIfAnyCount(rs);
+            }
+            else
+            {
+               rs.add(psli);
+            }
+         });
    }
 
 
@@ -347,4 +414,67 @@ public class ProcessSummaryWarningsAndErrorsRollup
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for uniqueErrorsToShow
+    *******************************************************************************/
+   public int getUniqueErrorsToShow()
+   {
+      return (this.uniqueErrorsToShow);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for uniqueErrorsToShow
+    *******************************************************************************/
+   public void setUniqueErrorsToShow(int uniqueErrorsToShow)
+   {
+      this.uniqueErrorsToShow = uniqueErrorsToShow;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for uniqueErrorsToShow
+    *******************************************************************************/
+   public ProcessSummaryWarningsAndErrorsRollup withUniqueErrorsToShow(int uniqueErrorsToShow)
+   {
+      this.uniqueErrorsToShow = uniqueErrorsToShow;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for doReplaceSingletonCountLinesWithSuffixOnly
+    *******************************************************************************/
+   public boolean getDoReplaceSingletonCountLinesWithSuffixOnly()
+   {
+      return (this.doReplaceSingletonCountLinesWithSuffixOnly);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for doReplaceSingletonCountLinesWithSuffixOnly
+    *******************************************************************************/
+   public void setDoReplaceSingletonCountLinesWithSuffixOnly(boolean doReplaceSingletonCountLinesWithSuffixOnly)
+   {
+      this.doReplaceSingletonCountLinesWithSuffixOnly = doReplaceSingletonCountLinesWithSuffixOnly;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for doReplaceSingletonCountLinesWithSuffixOnly
+    *******************************************************************************/
+   public ProcessSummaryWarningsAndErrorsRollup withDoReplaceSingletonCountLinesWithSuffixOnly(boolean doReplaceSingletonCountLinesWithSuffixOnly)
+   {
+      this.doReplaceSingletonCountLinesWithSuffixOnly = doReplaceSingletonCountLinesWithSuffixOnly;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java

@@ -95,7 +95,7 @@ public class ValueUtils
       }
       else if(value instanceof String s)
       {
-         return (Boolean.parseBoolean(s));
+         return "true".equalsIgnoreCase(s) || "yes".equalsIgnoreCase(s);
       }
       else
       {
@@ -496,6 +496,9 @@ public class ValueUtils
     *******************************************************************************/
    private static Instant tryAlternativeInstantParsing(String s, DateTimeParseException e)
    {
+      //////////////////////
+      // 1999-12-31T12:59 //
+      //////////////////////
       if(s.matches("^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}$"))
       {
          //////////////////////////
@@ -503,11 +506,34 @@ public class ValueUtils
          //////////////////////////
          return Instant.parse(s + ":00Z");
       }
+
+      ///////////////////////////
+      // 1999-12-31 12:59:59.0 //
+      ///////////////////////////
       else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.0$"))
       {
          s = s.replaceAll(" ", "T").replaceAll("\\..*$", "Z");
          return Instant.parse(s);
       }
+
+      /////////////////////////
+      // 1999-12-31 12:59:59 //
+      /////////////////////////
+      else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$"))
+      {
+         s = s.replaceAll(" ", "T") + "Z";
+         return Instant.parse(s);
+      }
+
+      //////////////////////
+      // 1999-12-31 12:59 //
+      //////////////////////
+      else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}$"))
+      {
+         s = s.replaceAll(" ", "T") + ":00Z";
+         return Instant.parse(s);
+      }
+
       else
       {
          try

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparator.java

@@ -0,0 +1,199 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.utils.collections;
+
+
+import java.util.Comparator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+
+/*******************************************************************************
+ ** Comparator for strings that are a mix of alpha + numeric, where we want to
+ ** sort the numeric substrings like numbers.
+ **
+ ** e.g., A1, A2, A10 won't come out as A1, A10, A2
+ *******************************************************************************/
+public class AlphaNumericComparator implements Comparator<String>
+{
+   private static final int A_FIRST = -1;
+   private static final int B_FIRST = 1;
+   private static final int TIE     = 0;
+
+   private static final Pattern INT_PATTERN            = Pattern.compile("^\\d+$");
+   private static final Pattern LEADING_INT_PATTERN    = Pattern.compile("^\\d+");
+   private static final Pattern ALPHA_THEN_INT_PATTERN = Pattern.compile("^(\\D+)\\d+");
+
+
+
+   /*******************************************************************************
+    ** compare 2 Strings
+    **
+    *******************************************************************************/
+   public int compare(String a, String b)
+   {
+      try
+      {
+         //////////////////////////////////////
+         // eliminate degenerate cases first //
+         //////////////////////////////////////
+         if(a == null && b == null)
+         {
+            return (TIE);
+         }
+         else if(a == null)
+         {
+            return (A_FIRST);
+         }
+         else if(b == null)
+         {
+            return (B_FIRST);
+         }
+         else if(a.equals(b))
+         {
+            return (TIE);
+         } // also covers a == "" and b == ""
+         else if(a.equals(""))
+         {
+            return (A_FIRST);
+         }
+         else if(b.equals(""))
+         {
+            return (B_FIRST);
+         }
+
+         ////////////////////////////////////////////////////////////////
+         // if both strings are pure numeric, parse as int and compare //
+         ////////////////////////////////////////////////////////////////
+         if(INT_PATTERN.matcher(a).matches() && INT_PATTERN.matcher(b).matches())
+         {
+            int intsCompared = new Integer(a).compareTo(new Integer(b));
+            if(intsCompared == TIE)
+            {
+               ///////////////////////////////////////////////////////////////////////////////
+               // in case the integers are the same (ie, "0001" vs "1"), compare as strings //
+               ///////////////////////////////////////////////////////////////////////////////
+               return (a.compareTo(b));
+            }
+            else
+            {
+               ///////////////////////////////////////////////////////////////
+               // else, if the ints were different, return their comparison //
+               ///////////////////////////////////////////////////////////////
+               return (intsCompared);
+            }
+         }
+
+         /////////////////////////////////////////////////////
+         // if both start as numbers, extract those numbers //
+         /////////////////////////////////////////////////////
+         Matcher aLeadingIntMatcher = LEADING_INT_PATTERN.matcher(a);
+         Matcher bLeadingIntMatcher = LEADING_INT_PATTERN.matcher(b);
+         if(aLeadingIntMatcher.lookingAt() && bLeadingIntMatcher.lookingAt())
+         {
+            ///////////////////////////
+            // extract the int parts //
+            ///////////////////////////
+            String aIntPart = a.substring(0, aLeadingIntMatcher.end());
+            String bIntPart = b.substring(0, bLeadingIntMatcher.end());
+
+            /////////////////////////////////////////////////////////////
+            // if the ints compare as non-zero, return that comparison //
+            /////////////////////////////////////////////////////////////
+            int intPartCompared = new Integer(aIntPart).compareTo(new Integer(bIntPart));
+            if(intPartCompared != TIE)
+            {
+               return (intPartCompared);
+            }
+            else
+            {
+               //////////////////////////////////////////////////////////////////////
+               // otherwise, make recursive call to compare the rest of the string //
+               //////////////////////////////////////////////////////////////////////
+               String aRest = a.substring(aLeadingIntMatcher.end());
+               String bRest = b.substring(bLeadingIntMatcher.end());
+               return (compare(aRest, bRest));
+            }
+         }
+         //////////////////////////////////////////////////////
+         // if one starts as numeric, but other doesn't      //
+         // return the one that starts with the number first //
+         //////////////////////////////////////////////////////
+         else if(aLeadingIntMatcher.lookingAt())
+         {
+            return (A_FIRST);
+         }
+         else if(bLeadingIntMatcher.lookingAt())
+         {
+            return (B_FIRST);
+         }
+
+         //////////////////////////////////////////////////////////////////////////
+         // now, if both parts have an alpha part, followed by digit parts, and  //
+         // the alpha parts are the same, then discard the alpha parts and recur //
+         //////////////////////////////////////////////////////////////////////////
+         Matcher aAlphaThenIntMatcher = ALPHA_THEN_INT_PATTERN.matcher(a);
+         Matcher bAlphaThenIntMatcher = ALPHA_THEN_INT_PATTERN.matcher(b);
+         if(aAlphaThenIntMatcher.lookingAt() && bAlphaThenIntMatcher.lookingAt())
+         {
+            String aAlphaPart = aAlphaThenIntMatcher.group(1);
+            String bAlphaPart = bAlphaThenIntMatcher.group(1);
+
+            if(aAlphaPart.equals(bAlphaPart))
+            {
+               String aRest = a.substring(aAlphaPart.length());
+               String bRest = b.substring(bAlphaPart.length());
+               return (compare(aRest, bRest));
+            }
+         }
+
+         /////////////////////////////////////////////////
+         // as last resort, just do pure string compare //
+         /////////////////////////////////////////////////
+         return (a.compareTo(b));
+      }
+      catch(Exception e)
+      {
+         //////////////////////////////////////////////////////////
+         // on exception, don't allow caller to catch -- rather, //
+         // always return something sensible (and null-safe)     //
+         //////////////////////////////////////////////////////////
+         if(a == null && b == null)
+         {
+            return (TIE);
+         }
+         else if(a == null)
+         {
+            return (A_FIRST);
+         }
+         else if(b == null)
+         {
+            return (B_FIRST);
+         }
+         else
+         {
+            return (a.compareTo(b));
+         }
+      }
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java

@@ -22,15 +22,19 @@
 package com.kingsrook.qqq.backend.core.adapters;
 
 
+import java.time.LocalDate;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QIndexBasedFieldMapping;
 import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
@@ -395,4 +399,70 @@ class CsvToQRecordAdapterTest extends BaseTest
       assertEquals("john@doe.com", records.get(0).getValueString("email"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildRecordsFromCsv_doCorrectValueTypes() throws QException
+   {
+      CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+      csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
+         .withDoCorrectValueTypes(true)
+         .withTable(TestUtils.defineTablePerson().withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN)))
+         .withCsv("""
+            firstName,birthDate,isEmployed
+            John,1/1/1980,true
+            Paul,1970-06-15,Yes
+            George,,anything-else
+            """));
+      List<QRecord> qRecords = csvToQRecordAdapter.getRecordList();
+
+      QRecord qRecord = qRecords.get(0);
+      assertEquals("John", qRecord.getValue("firstName"));
+      assertEquals(LocalDate.parse("1980-01-01"), qRecord.getValue("birthDate"));
+      assertEquals(true, qRecord.getValue("isEmployed"));
+
+      qRecord = qRecords.get(1);
+      assertEquals("Paul", qRecord.getValue("firstName"));
+      assertEquals(LocalDate.parse("1970-06-15"), qRecord.getValue("birthDate"));
+      assertEquals(true, qRecord.getValue("isEmployed"));
+
+      qRecord = qRecords.get(2);
+      assertEquals("George", qRecord.getValue("firstName"));
+      assertNull(qRecord.getValue("birthDate"));
+      assertEquals(false, qRecord.getValue("isEmployed"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildRecordsFromCsv_doCorrectValueTypesErrorsForUnparseable() throws QException
+   {
+      CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+      csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
+         .withDoCorrectValueTypes(true)
+         .withTable(TestUtils.defineTablePerson())
+         .withCsv("""
+            firstName,birthDate,favoriteShapeId
+            John,1980,1
+            Paul,1970-06-15,green
+            """));
+      List<QRecord> qRecords = csvToQRecordAdapter.getRecordList();
+
+      QRecord qRecord = qRecords.get(0);
+      assertEquals("John", qRecord.getValue("firstName"));
+      assertThat(qRecord.getErrors()).hasSize(1);
+      assertThat(qRecord.getErrors().get(0).toString()).isEqualTo("Error parsing line #1: Could not parse value [1980] to a local date");
+
+      qRecord = qRecords.get(1);
+      assertEquals("Paul", qRecord.getValue("firstName"));
+      assertThat(qRecord.getErrors()).hasSize(1);
+      assertThat(qRecord.getErrors().get(0).toString()).isEqualTo("Error parsing line #2: Value [green] could not be converted to an Integer.");
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java

@@ -62,6 +62,42 @@ class BulkInsertTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String getPersonCsvRow1()
+   {
+      return ("""
+         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String getPersonCsvRow2()
+   {
+      return ("""
+         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String getPersonCsvHeaderUsingLabels()
+   {
+      return ("""
+         "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email"
+         """);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -77,7 +113,7 @@ class BulkInsertTest extends BaseTest
       // create an uploaded file, similar to how an http server may //
       ////////////////////////////////////////////////////////////////
       QUploadedFile qUploadedFile = new QUploadedFile();
-      qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes());
+      qUploadedFile.setBytes((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes());
       qUploadedFile.setFilename("test.csv");
       UUIDAndTypeStateKey uploadedFileKey = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE);
       TempFileStateProvider.getInstance().put(uploadedFileKey, qUploadedFile);

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -1252,41 +1252,6 @@ public class TestUtils
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static String getPersonCsvHeaderUsingLabels()
-   {
-      return ("""
-         "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email"
-         """);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static String getPersonCsvRow1()
-   {
-      return ("""
-         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"
-         """);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static String getPersonCsvRow2()
-   {
-      return ("""
-         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"
-         """);
-   }
-
-
 
    /*******************************************************************************
     **

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparatorTest.java

@@ -0,0 +1,149 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.utils.collections;
+
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for AlphaNumericComparator 
+ *******************************************************************************/
+class AlphaNumericComparatorTest extends BaseTest
+{
+
+   /*******************************************************************************
+    ** test odd-balls
+    **
+    *******************************************************************************/
+   @Test
+   public void testFringeCases()
+   {
+      test(sort("", null, "foo", " ", "", "1", null),
+         null, null, "", "", "1", " ", "foo");
+   }
+
+
+
+   /*******************************************************************************
+    ** test alpha-strings only
+    **
+    *******************************************************************************/
+   @Test
+   public void testAlphasOnly()
+   {
+      test(sort("F", "G", "A", "AB", "BB", "BA", "BD"),
+         "A", "AB", "BA", "BB", "BD", "F", "G");
+   }
+
+
+
+   /*******************************************************************************
+    ** test numbers only
+    **
+    *******************************************************************************/
+   @Test
+   public void testNumbersOnly()
+   {
+      test(sort("1", "273", "271", "102", "101", "10", "13", "2", "22", "273"),
+         "1", "2", "10", "13", "22", "101", "102", "271", "273", "273");
+   }
+
+
+
+   /*******************************************************************************
+    ** test mixed
+    **
+    *******************************************************************************/
+   @Test
+   public void testMixed1()
+   {
+      test(sort("1", "A", "A1", "1A", "10", "10AA", "11", "A11", "11B", "1B", "A10B2", "A10B10", "D1", "D10", "D2", "F20G11H10", "F3", "F20G11H2", "A1", "A10", "A2", "01", "001"),
+         "001", "01", "1", "1A", "1B", "10", "10AA", "11", "11B", "A", "A1", "A1", "A2", "A10", "A10B2", "A10B10", "A11", "D1", "D2", "D10", "F3", "F20G11H2", "F20G11H10");
+   }
+
+
+
+   /*******************************************************************************
+    ** test mixed
+    **
+    *******************************************************************************/
+   @Test
+   public void testMixed2()
+   {
+      test(sort("A", "A001", "A1", "A0000", "A00001", "000023", "023", "000023", "023A", "23", "2", "0002", "02"),
+         "0002", "02", "2", "000023", "000023", "023", "23", "023A", "A", "A0000", "A00001", "A001", "A1");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    **
+    *******************************************************************************/
+   private void test(List<String> a, String... b)
+   {
+      System.out.println("Expecting: " + Arrays.asList(b));
+
+      assertEquals(a.size(), b.length);
+
+      for(int i = 0; i < a.size(); i++)
+      {
+         String aString = a.get(i);
+         String bString = b[i];
+
+         assertEquals(aString, bString);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    **
+    *******************************************************************************/
+   private List<String> sort(String... input)
+   {
+      List<String> inputList = Arrays.asList(input);
+      System.out.println("Sorting:   " + inputList);
+
+      try
+      {
+         List<String> naturalSortList = Arrays.asList(input);
+         Collections.sort(naturalSortList);
+         System.out.println("Natural:   " + naturalSortList);
+      }
+      catch(Exception e)
+      {
+         System.out.println("Natural:   FAILED");
+      }
+
+      inputList.sort(new AlphaNumericComparator());
+      System.out.println("Produced:  " + inputList);
+      return (inputList);
+   }
+}