QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improving process traces built by bulk load

Browse Source
This commit is contained in:
darin.kelkhoff
2025-04-21 10:58:56 -05:00
parent 97434ebb66
commit f81b257dd4
3 changed files with 17 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java
View File

@ -55,8 +55,6 @@ public class BulkInsertExtractStep extends AbstractExtractStep
@Override @Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{ {
runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput));
int rowsAdded = 0; int rowsAdded = 0;
int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE); int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE);

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java
View File

@ -52,6 +52,11 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep
@Override @Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{ {
////////////////////////////////////////////////////////////////////////////////////////
// for headless-bulk load (e.g., sftp import), set up the process tracer's key record //
////////////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput));
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if user has come back here, clear out file (else the storageInput object that it is comes to the frontend, which isn't what we want!) // // if user has come back here, clear out file (else the storageInput object that it is comes to the frontend, which isn't what we want!) //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

14
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java
View File

@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mode
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.BooleanUtils;
@ -77,10 +78,14 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
if(savedBulkLoadProfileRecord == null) if(savedBulkLoadProfileRecord == null)
{ {
throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load")); throw (new QUserFacingException("Did not receive a Bulk Load Profile record as input. Unable to perform headless bulk load"));
} }
SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord); SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord);
if(!StringUtils.hasContent(savedBulkLoadProfile.getMappingJson()))
{
throw (new QUserFacingException("Bulk Load Profile record's Mapping is empty. Unable to perform headless bulk load"));
}
try try
{ {
@ -88,7 +93,7 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
} }
catch(Exception e) catch(Exception e)
{ {
throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e)); throw (new QUserFacingException("Error processing Bulk Load Profile record. Unable to perform headless bulk load", e));
} }
} }
else else
@ -240,6 +245,11 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
} }
} }
} }
catch(QUserFacingException ufe)
{
LOG.warn("User-facing error in bulk insert receive mapping", ufe);
throw ufe;
}
catch(Exception e) catch(Exception e)
{ {
LOG.warn("Error in bulk insert receive mapping", e); LOG.warn("Error in bulk insert receive mapping", e);