From a840bd1d508a9b3ddc589acdeeade8ab9e40d892 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 15 Aug 2022 10:55:23 -0500 Subject: [PATCH] Adding status updates to ETL Load; Add YYYYmmDD as localDate format --- .../etl/basic/BasicETLLoadAsUpdateFunction.java | 2 ++ .../implementations/etl/basic/BasicETLLoadFunction.java | 2 ++ .../java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadAsUpdateFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadAsUpdateFunction.java index e6a8290f..0219acf6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadAsUpdateFunction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadAsUpdateFunction.java @@ -82,6 +82,8 @@ public class BasicETLLoadAsUpdateFunction implements BackendStep for(List page : CollectionUtils.getPages(inputRecords, pageSize)) { LOG.info("Updating a page of [" + page.size() + "] records. Progress: " + recordsUpdated + " loaded out of " + inputRecords.size() + " total"); + runBackendStepInput.getAsyncJobCallback().updateStatus("Updating records", recordsUpdated, inputRecords.size()); + UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance()); updateInput.setSession(runBackendStepInput.getSession()); updateInput.setTableName(table); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java index f8e782f2..d0a6c77b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java @@ -86,6 +86,8 @@ public class BasicETLLoadFunction implements BackendStep for(List page : CollectionUtils.getPages(inputRecords, pageSize)) { LOG.info("Inserting a page of [" + page.size() + "] records. Progress: " + recordsInserted + " loaded out of " + inputRecords.size() + " total"); + runBackendStepInput.getAsyncJobCallback().updateStatus("Inserting records", recordsInserted, inputRecords.size()); + InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance()); insertInput.setSession(runBackendStepInput.getSession()); insertInput.setTableName(table); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java index 08ecf054..0dbe02b3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java @@ -44,6 +44,7 @@ public class ValueUtils { private static final DateTimeFormatter dateTimeFormatter_yyyyMMddWithDashes = DateTimeFormatter.ofPattern("yyyy-MM-dd"); private static final DateTimeFormatter dateTimeFormatter_MdyyyyWithSlashes = DateTimeFormatter.ofPattern("M/d/yyyy"); + private static final DateTimeFormatter dateTimeFormatter_yyyyMMdd = DateTimeFormatter.ofPattern("yyyyMMdd"); @@ -262,7 +263,7 @@ public class ValueUtils private static LocalDate tryLocalDateParsers(String s) { DateTimeParseException lastException = null; - for(DateTimeFormatter dateTimeFormatter : List.of(dateTimeFormatter_yyyyMMddWithDashes, dateTimeFormatter_MdyyyyWithSlashes)) + for(DateTimeFormatter dateTimeFormatter : List.of(dateTimeFormatter_yyyyMMddWithDashes, dateTimeFormatter_MdyyyyWithSlashes, dateTimeFormatter_yyyyMMdd )) { try {