diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java index dabfb3f4..694ae332 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java @@ -55,8 +55,6 @@ public class BulkInsertExtractStep extends AbstractExtractStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput)); - int rowsAdded = 0; int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java index 30f98c91..c60503fe 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -77,19 +78,77 @@ public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSumm QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getValueString("tableName")); + ///////////////////////////////////////////////////////////////////////////////////////////// + // the transform step builds summary lines that it predicts will insert successfully. // + // but those lines don't have ids, which we'd like to have (e.g., for a process trace that // + // might link to the built record). also, it's possible that there was a fail that only // + // happened in the actual insert, so, basically, re-do the summary here // + ///////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertTransformStep transformStep = (BulkInsertTransformStep) getTransformStep(); + ProcessSummaryLine okSummary = transformStep.okSummary; + okSummary.setCount(0); + okSummary.setPrimaryKeys(new ArrayList<>()); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // but - since errors from the transform step don't even make it through to us here in the load step, // + // do re-use the ProcessSummaryWarningsAndErrorsRollup from transform step as follows: // + // clear out its warnings - we'll completely rebuild them here (with primary keys) // + // and add new error lines, e.g., in case of errors that only happened past the validation if possible. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = transformStep.processSummaryWarningsAndErrorsRollup; + processSummaryWarningsAndErrorsRollup.resetWarnings(); + List insertedRecords = runBackendStepOutput.getRecords(); for(QRecord insertedRecord : insertedRecords) { - if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors())) + Serializable primaryKey = insertedRecord.getValue(table.getPrimaryKeyField()); + if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()) && primaryKey != null) { + ///////////////////////////////////////////////////////////////////////// + // if the record had no errors, and we have a primary key for it, then // + // keep track of the range of primary keys (first and last) // + ///////////////////////////////////////////////////////////////////////// if(firstInsertedPrimaryKey == null) { - firstInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField()); + firstInsertedPrimaryKey = primaryKey; } - lastInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField()); + lastInsertedPrimaryKey = primaryKey; + + if(!CollectionUtils.nullSafeIsEmpty(insertedRecord.getWarnings())) + { + ///////////////////////////////////////////////////////////////////////////// + // if there were warnings on the inserted record, put it in a warning line // + ///////////////////////////////////////////////////////////////////////////// + String message = insertedRecord.getWarnings().get(0).getMessage(); + processSummaryWarningsAndErrorsRollup.addWarning(message, primaryKey); + } + else + { + //////////////////////////////////////////////////////////////////////// + // if no warnings for the inserted record, then put it in the OK line // + //////////////////////////////////////////////////////////////////////// + okSummary.incrementCountAndAddPrimaryKey(primaryKey); + } + } + else + { + ////////////////////////////////////////////////////////////////////// + // else if there were errors or no primary key, build an error line // + ////////////////////////////////////////////////////////////////////// + String message = "Failed to insert"; + if(!CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors())) + { + ////////////////////////////////////////////////////////// + // use the error message from the record if we have one // + ////////////////////////////////////////////////////////// + message = insertedRecord.getErrors().get(0).getMessage(); + } + processSummaryWarningsAndErrorsRollup.addError(message, primaryKey); } } + + okSummary.pickMessage(true); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java index ea4810a9..4b15b720 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java @@ -52,6 +52,11 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { + //////////////////////////////////////////////////////////////////////////////////////// + // for headless-bulk load (e.g., sftp import), set up the process tracer's key record // + //////////////////////////////////////////////////////////////////////////////////////// + runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput)); + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if user has come back here, clear out file (else the storageInput object that it is comes to the frontend, which isn't what we want!) // /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java index 94032fe5..aa460540 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java @@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mode import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.commons.lang3.BooleanUtils; @@ -77,10 +78,14 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep ////////////////////////////////////////////////////////////////////////////// if(savedBulkLoadProfileRecord == null) { - throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load")); + throw (new QUserFacingException("Did not receive a Bulk Load Profile record as input. Unable to perform headless bulk load")); } SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord); + if(!StringUtils.hasContent(savedBulkLoadProfile.getMappingJson())) + { + throw (new QUserFacingException("Bulk Load Profile record's Mapping is empty. Unable to perform headless bulk load")); + } try { @@ -88,7 +93,7 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep } catch(Exception e) { - throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e)); + throw (new QUserFacingException("Error processing Bulk Load Profile record. Unable to perform headless bulk load", e)); } } else @@ -240,6 +245,11 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep } } } + catch(QUserFacingException ufe) + { + LOG.warn("User-facing error in bulk insert receive mapping", ufe); + throw ufe; + } catch(Exception e) { LOG.warn("Error in bulk insert receive mapping", e); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 410fccf2..e3979a07 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -75,9 +75,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public class BulkInsertTransformStep extends AbstractTransformStep { - private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); - private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted") + ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted") .withDoReplaceSingletonCountLinesWithSuffixOnly(false); private ListingHash errorToExampleRowValueMap = new ListingHash<>(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java index 6296f31c..79dbe9ca 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java @@ -195,7 +195,7 @@ public class ProcessSummaryWarningsAndErrorsRollup { if(otherWarningsSummary == null) { - otherWarningsSummary = new ProcessSummaryLine(Status.WARNING).withMessageSuffix("records had an other warning."); + otherWarningsSummary = buildOtherWarningsSummary(); } processSummaryLine = otherWarningsSummary; } @@ -214,6 +214,27 @@ public class ProcessSummaryWarningsAndErrorsRollup + /*************************************************************************** + ** + ***************************************************************************/ + private static ProcessSummaryLine buildOtherWarningsSummary() + { + return new ProcessSummaryLine(Status.WARNING).withMessageSuffix("records had an other warning."); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public void resetWarnings() + { + warningSummaries.clear(); + otherWarningsSummary = buildOtherWarningsSummary(); + } + + + /******************************************************************************* ** Wrapper around AlphaNumericComparator for ProcessSummaryLineInterface that ** extracts string messages out. diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java index 60d4561c..59b79181 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java @@ -185,4 +185,13 @@ public class ProcessSummaryLineInterfaceAssert extends AbstractAssert preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + for(QRecord record : records) + { + if(record.getValueString("firstName").toLowerCase().contains("warn")) + { + record.addWarning(new QWarningMessage(record.getValueString("firstName"))); + } + else if(record.getValueString("firstName").toLowerCase().contains("error")) + { + if(isPreview && record.getValueString("firstName").toLowerCase().contains("not-pre-error")) + { + continue; + } + + record.addError(new BadInputStatusMessage(record.getValueString("firstName"))); + } + } + return records; + } + } } \ No newline at end of file