From f81b257dd44f1366b3565f4e73c62595212ad17b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 21 Apr 2025 10:58:56 -0500 Subject: [PATCH] Improving process traces built by bulk load --- .../bulk/insert/BulkInsertExtractStep.java | 2 -- .../insert/BulkInsertPrepareFileUploadStep.java | 5 +++++ .../insert/BulkInsertReceiveFileMappingStep.java | 14 ++++++++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java index dabfb3f4..694ae332 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java @@ -55,8 +55,6 @@ public class BulkInsertExtractStep extends AbstractExtractStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput)); - int rowsAdded = 0; int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java index ea4810a9..4b15b720 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java @@ -52,6 +52,11 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { + //////////////////////////////////////////////////////////////////////////////////////// + // for headless-bulk load (e.g., sftp import), set up the process tracer's key record // + //////////////////////////////////////////////////////////////////////////////////////// + runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput)); + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if user has come back here, clear out file (else the storageInput object that it is comes to the frontend, which isn't what we want!) // /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java index 94032fe5..aa460540 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java @@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mode import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.commons.lang3.BooleanUtils; @@ -77,10 +78,14 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep ////////////////////////////////////////////////////////////////////////////// if(savedBulkLoadProfileRecord == null) { - throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load")); + throw (new QUserFacingException("Did not receive a Bulk Load Profile record as input. Unable to perform headless bulk load")); } SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord); + if(!StringUtils.hasContent(savedBulkLoadProfile.getMappingJson())) + { + throw (new QUserFacingException("Bulk Load Profile record's Mapping is empty. Unable to perform headless bulk load")); + } try { @@ -88,7 +93,7 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep } catch(Exception e) { - throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e)); + throw (new QUserFacingException("Error processing Bulk Load Profile record. Unable to perform headless bulk load", e)); } } else @@ -240,6 +245,11 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep } } } + catch(QUserFacingException ufe) + { + LOG.warn("User-facing error in bulk insert receive mapping", ufe); + throw ufe; + } catch(Exception e) { LOG.warn("Error in bulk insert receive mapping", e);