From a840bd1d508a9b3ddc589acdeeade8ab9e40d892 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 15 Aug 2022 10:55:23 -0500 Subject: [PATCH 1/4] Adding status updates to ETL Load; Add YYYYmmDD as localDate format --- .../etl/basic/BasicETLLoadAsUpdateFunction.java | 2 ++ .../implementations/etl/basic/BasicETLLoadFunction.java | 2 ++ .../java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadAsUpdateFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadAsUpdateFunction.java index e6a8290f..0219acf6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadAsUpdateFunction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadAsUpdateFunction.java @@ -82,6 +82,8 @@ public class BasicETLLoadAsUpdateFunction implements BackendStep for(List page : CollectionUtils.getPages(inputRecords, pageSize)) { LOG.info("Updating a page of [" + page.size() + "] records. Progress: " + recordsUpdated + " loaded out of " + inputRecords.size() + " total"); + runBackendStepInput.getAsyncJobCallback().updateStatus("Updating records", recordsUpdated, inputRecords.size()); + UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance()); updateInput.setSession(runBackendStepInput.getSession()); updateInput.setTableName(table); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java index f8e782f2..d0a6c77b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java @@ -86,6 +86,8 @@ public class BasicETLLoadFunction implements BackendStep for(List page : CollectionUtils.getPages(inputRecords, pageSize)) { LOG.info("Inserting a page of [" + page.size() + "] records. Progress: " + recordsInserted + " loaded out of " + inputRecords.size() + " total"); + runBackendStepInput.getAsyncJobCallback().updateStatus("Inserting records", recordsInserted, inputRecords.size()); + InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance()); insertInput.setSession(runBackendStepInput.getSession()); insertInput.setTableName(table); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java index 08ecf054..0dbe02b3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java @@ -44,6 +44,7 @@ public class ValueUtils { private static final DateTimeFormatter dateTimeFormatter_yyyyMMddWithDashes = DateTimeFormatter.ofPattern("yyyy-MM-dd"); private static final DateTimeFormatter dateTimeFormatter_MdyyyyWithSlashes = DateTimeFormatter.ofPattern("M/d/yyyy"); + private static final DateTimeFormatter dateTimeFormatter_yyyyMMdd = DateTimeFormatter.ofPattern("yyyyMMdd"); @@ -262,7 +263,7 @@ public class ValueUtils private static LocalDate tryLocalDateParsers(String s) { DateTimeParseException lastException = null; - for(DateTimeFormatter dateTimeFormatter : List.of(dateTimeFormatter_yyyyMMddWithDashes, dateTimeFormatter_MdyyyyWithSlashes)) + for(DateTimeFormatter dateTimeFormatter : List.of(dateTimeFormatter_yyyyMMddWithDashes, dateTimeFormatter_MdyyyyWithSlashes, dateTimeFormatter_yyyyMMdd )) { try { From 5735bdf9d7cbf4abde8bb2445c190497ea72ffc5 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 15 Aug 2022 10:55:39 -0500 Subject: [PATCH 2/4] Update to 0.3.1-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f368b909..ec6dbc2f 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ - 0.3.0 + 0.3.1-SNAPSHOT UTF-8 UTF-8 From a0cfd5a97eb42959a93621a91498909c5e621664 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 15 Aug 2022 10:58:10 -0500 Subject: [PATCH 3/4] Checkstyle fix --- .../java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java index 0dbe02b3..9850719b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java @@ -44,7 +44,7 @@ public class ValueUtils { private static final DateTimeFormatter dateTimeFormatter_yyyyMMddWithDashes = DateTimeFormatter.ofPattern("yyyy-MM-dd"); private static final DateTimeFormatter dateTimeFormatter_MdyyyyWithSlashes = DateTimeFormatter.ofPattern("M/d/yyyy"); - private static final DateTimeFormatter dateTimeFormatter_yyyyMMdd = DateTimeFormatter.ofPattern("yyyyMMdd"); + private static final DateTimeFormatter dateTimeFormatter_yyyyMMdd = DateTimeFormatter.ofPattern("yyyyMMdd"); @@ -263,7 +263,7 @@ public class ValueUtils private static LocalDate tryLocalDateParsers(String s) { DateTimeParseException lastException = null; - for(DateTimeFormatter dateTimeFormatter : List.of(dateTimeFormatter_yyyyMMddWithDashes, dateTimeFormatter_MdyyyyWithSlashes, dateTimeFormatter_yyyyMMdd )) + for(DateTimeFormatter dateTimeFormatter : List.of(dateTimeFormatter_yyyyMMddWithDashes, dateTimeFormatter_MdyyyyWithSlashes, dateTimeFormatter_yyyyMMdd)) { try { From 9bf898af7a74806b9bf63b7c7ff7dd2ccc7a7c54 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 17 Aug 2022 11:34:00 -0500 Subject: [PATCH 4/4] Update to handle BOM char and index-out-of-bounds condition --- .../core/adapters/CsvToQRecordAdapter.java | 11 +++- .../adapters/CsvToQRecordAdapterTest.java | 57 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java index 3d5493a3..98f85478 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java @@ -95,6 +95,15 @@ public class CsvToQRecordAdapter throw (new IllegalArgumentException("Empty csv value was provided.")); } + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // once, from a DOS csv file (that had come from Excel), we had a "" character (FEFF, Byte-order marker) at the start of a // + // CSV, which caused our first header to not match... So, let us strip away any FEFF or FFFE's at the start of CSV strings. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(csv.length() > 1 && (csv.charAt(0) == 0xfeff || csv.charAt(0) == 0xfffe)) + { + csv = csv.substring(1); + } + try { /////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -118,7 +127,7 @@ public class CsvToQRecordAdapter // put values from the CSV record into a map of header -> value // ////////////////////////////////////////////////////////////////// Map csvValues = new HashMap<>(); - for(int i = 0; i < headers.size(); i++) + for(int i = 0; i < headers.size() && i < csvRecord.size(); i++) { csvValues.put(headers.get(i), csvRecord.get(i)); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java index 2593b3e2..a81e5b53 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java @@ -31,6 +31,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -281,4 +282,60 @@ class CsvToQRecordAdapterTest // todo - this is what the method header comment means when it says we don't handle all cases well... // Assertions.assertEquals(List.of("A", "B", "C", "C 2", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C 2", "C", "C 3"))); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testByteOrderMarker() + { + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + List records = csvToQRecordAdapter.buildRecordsFromCsv(""" + id,firstName + 1,John""", TestUtils.defineTablePerson(), null); + + assertEquals(1, records.get(0).getValueInteger("id")); + assertEquals("John", records.get(0).getValueString("firstName")); + } + + + + /******************************************************************************* + ** Fix an IndexOutOfBounds that we used to throw. + *******************************************************************************/ + @Test + void testTooFewBodyColumns() + { + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + List records = csvToQRecordAdapter.buildRecordsFromCsv(""" + id,firstName,lastName + 1,John""", TestUtils.defineTablePerson(), null); + + assertEquals(1, records.get(0).getValueInteger("id")); + assertEquals("John", records.get(0).getValueString("firstName")); + assertNull(records.get(0).getValueString("lastName")); + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testTooFewColumnsIndexMapping() + { + int index = 1; + QIndexBasedFieldMapping mapping = new QIndexBasedFieldMapping() + .withMapping("id", index++) + .withMapping("firstName", index++) + .withMapping("lastName", index++); + + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + List records = csvToQRecordAdapter.buildRecordsFromCsv("1,John", TestUtils.defineTablePerson(), mapping); + + assertEquals(1, records.get(0).getValueInteger("id")); + assertEquals("John", records.get(0).getValueString("firstName")); + assertNull(records.get(0).getValueString("lastName")); + } + }