CE-604 Bulk load improvements - type handling (via csv adapter) and better errors

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java

@@ -22,15 +22,19 @@
 package com.kingsrook.qqq.backend.core.adapters;
 
 
+import java.time.LocalDate;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QIndexBasedFieldMapping;
 import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
@@ -395,4 +399,70 @@ class CsvToQRecordAdapterTest extends BaseTest
       assertEquals("john@doe.com", records.get(0).getValueString("email"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildRecordsFromCsv_doCorrectValueTypes() throws QException
+   {
+      CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+      csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
+         .withDoCorrectValueTypes(true)
+         .withTable(TestUtils.defineTablePerson().withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN)))
+         .withCsv("""
+            firstName,birthDate,isEmployed
+            John,1/1/1980,true
+            Paul,1970-06-15,Yes
+            George,,anything-else
+            """));
+      List<QRecord> qRecords = csvToQRecordAdapter.getRecordList();
+
+      QRecord qRecord = qRecords.get(0);
+      assertEquals("John", qRecord.getValue("firstName"));
+      assertEquals(LocalDate.parse("1980-01-01"), qRecord.getValue("birthDate"));
+      assertEquals(true, qRecord.getValue("isEmployed"));
+
+      qRecord = qRecords.get(1);
+      assertEquals("Paul", qRecord.getValue("firstName"));
+      assertEquals(LocalDate.parse("1970-06-15"), qRecord.getValue("birthDate"));
+      assertEquals(true, qRecord.getValue("isEmployed"));
+
+      qRecord = qRecords.get(2);
+      assertEquals("George", qRecord.getValue("firstName"));
+      assertNull(qRecord.getValue("birthDate"));
+      assertEquals(false, qRecord.getValue("isEmployed"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildRecordsFromCsv_doCorrectValueTypesErrorsForUnparseable() throws QException
+   {
+      CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+      csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
+         .withDoCorrectValueTypes(true)
+         .withTable(TestUtils.defineTablePerson())
+         .withCsv("""
+            firstName,birthDate,favoriteShapeId
+            John,1980,1
+            Paul,1970-06-15,green
+            """));
+      List<QRecord> qRecords = csvToQRecordAdapter.getRecordList();
+
+      QRecord qRecord = qRecords.get(0);
+      assertEquals("John", qRecord.getValue("firstName"));
+      assertThat(qRecord.getErrors()).hasSize(1);
+      assertThat(qRecord.getErrors().get(0).toString()).isEqualTo("Error parsing line #1: Could not parse value [1980] to a local date");
+
+      qRecord = qRecords.get(1);
+      assertEquals("Paul", qRecord.getValue("firstName"));
+      assertThat(qRecord.getErrors()).hasSize(1);
+      assertThat(qRecord.getErrors().get(0).toString()).isEqualTo("Error parsing line #2: Value [green] could not be converted to an Integer.");
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java

@@ -62,6 +62,42 @@ class BulkInsertTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String getPersonCsvRow1()
+   {
+      return ("""
+         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String getPersonCsvRow2()
+   {
+      return ("""
+         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String getPersonCsvHeaderUsingLabels()
+   {
+      return ("""
+         "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email"
+         """);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -77,7 +113,7 @@ class BulkInsertTest extends BaseTest
       // create an uploaded file, similar to how an http server may //
       ////////////////////////////////////////////////////////////////
       QUploadedFile qUploadedFile = new QUploadedFile();
-      qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes());
+      qUploadedFile.setBytes((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes());
       qUploadedFile.setFilename("test.csv");
       UUIDAndTypeStateKey uploadedFileKey = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE);
       TempFileStateProvider.getInstance().put(uploadedFileKey, qUploadedFile);

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -1252,41 +1252,6 @@ public class TestUtils
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static String getPersonCsvHeaderUsingLabels()
-   {
-      return ("""
-         "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email"
-         """);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static String getPersonCsvRow1()
-   {
-      return ("""
-         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"
-         """);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static String getPersonCsvRow2()
-   {
-      return ("""
-         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"
-         """);
-   }
-
-
 
    /*******************************************************************************
     **

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparatorTest.java

@@ -0,0 +1,149 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.utils.collections;
+
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for AlphaNumericComparator 
+ *******************************************************************************/
+class AlphaNumericComparatorTest extends BaseTest
+{
+
+   /*******************************************************************************
+    ** test odd-balls
+    **
+    *******************************************************************************/
+   @Test
+   public void testFringeCases()
+   {
+      test(sort("", null, "foo", " ", "", "1", null),
+         null, null, "", "", "1", " ", "foo");
+   }
+
+
+
+   /*******************************************************************************
+    ** test alpha-strings only
+    **
+    *******************************************************************************/
+   @Test
+   public void testAlphasOnly()
+   {
+      test(sort("F", "G", "A", "AB", "BB", "BA", "BD"),
+         "A", "AB", "BA", "BB", "BD", "F", "G");
+   }
+
+
+
+   /*******************************************************************************
+    ** test numbers only
+    **
+    *******************************************************************************/
+   @Test
+   public void testNumbersOnly()
+   {
+      test(sort("1", "273", "271", "102", "101", "10", "13", "2", "22", "273"),
+         "1", "2", "10", "13", "22", "101", "102", "271", "273", "273");
+   }
+
+
+
+   /*******************************************************************************
+    ** test mixed
+    **
+    *******************************************************************************/
+   @Test
+   public void testMixed1()
+   {
+      test(sort("1", "A", "A1", "1A", "10", "10AA", "11", "A11", "11B", "1B", "A10B2", "A10B10", "D1", "D10", "D2", "F20G11H10", "F3", "F20G11H2", "A1", "A10", "A2", "01", "001"),
+         "001", "01", "1", "1A", "1B", "10", "10AA", "11", "11B", "A", "A1", "A1", "A2", "A10", "A10B2", "A10B10", "A11", "D1", "D2", "D10", "F3", "F20G11H2", "F20G11H10");
+   }
+
+
+
+   /*******************************************************************************
+    ** test mixed
+    **
+    *******************************************************************************/
+   @Test
+   public void testMixed2()
+   {
+      test(sort("A", "A001", "A1", "A0000", "A00001", "000023", "023", "000023", "023A", "23", "2", "0002", "02"),
+         "0002", "02", "2", "000023", "000023", "023", "23", "023A", "A", "A0000", "A00001", "A001", "A1");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    **
+    *******************************************************************************/
+   private void test(List<String> a, String... b)
+   {
+      System.out.println("Expecting: " + Arrays.asList(b));
+
+      assertEquals(a.size(), b.length);
+
+      for(int i = 0; i < a.size(); i++)
+      {
+         String aString = a.get(i);
+         String bString = b[i];
+
+         assertEquals(aString, bString);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    **
+    *******************************************************************************/
+   private List<String> sort(String... input)
+   {
+      List<String> inputList = Arrays.asList(input);
+      System.out.println("Sorting:   " + inputList);
+
+      try
+      {
+         List<String> naturalSortList = Arrays.asList(input);
+         Collections.sort(naturalSortList);
+         System.out.println("Natural:   " + naturalSortList);
+      }
+      catch(Exception e)
+      {
+         System.out.println("Natural:   FAILED");
+      }
+
+      inputList.sort(new AlphaNumericComparator());
+      System.out.println("Produced:  " + inputList);
+      return (inputList);
+   }
+}