QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Start (mostly done?) support for headless bulk-load

Browse Source
This commit is contained in:
darin.kelkhoff
2025-02-14 20:30:13 -06:00
parent b93114a9ba
commit c341708d21
2 changed files with 94 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java
View File

@ -32,16 +32,20 @@ import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang3.BooleanUtils;
@ -63,12 +67,37 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
{
try
{
BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
QRecord savedBulkLoadProfileRecord = BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
///////////////////////////////////////////////////////////////////
// read process values - construct a bulkLoadProfile out of them //
///////////////////////////////////////////////////////////////////
BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput);
BulkLoadProfile bulkLoadProfile;
if(BulkInsertStepUtils.isHeadless(runBackendStepInput))
{
//////////////////////////////////////////////////////////////////////////////
// if running headless, build bulkLoadProfile from the saved profile record //
//////////////////////////////////////////////////////////////////////////////
if(savedBulkLoadProfileRecord == null)
{
throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load"));
}
SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord);
try
{
bulkLoadProfile = JsonUtils.toObject(savedBulkLoadProfile.getMappingJson(), BulkLoadProfile.class);
}
catch(Exception e)
{
throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e));
}
}
else
{
///////////////////////////////////////////////////////////////////
// read process values - construct a bulkLoadProfile out of them //
///////////////////////////////////////////////////////////////////
bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput);
}
/////////////////////////////////////////////////////////////////////////
// put the list of bulk load profile into the process state - it's the //
@ -183,21 +212,32 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
/////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping);
if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping))
if(BulkInsertStepUtils.isHeadless(runBackendStepInput))
{
//////////////////////////////////////////////////////////////////////////////////
// just go to the prepareValueMapping backend step - it'll figure out the rest. //
// it's also where the value-mapping loop of steps points. //
// and, this will actually be the default (e.g., the step after this one). //
//////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.addValue("valueMappingFieldIndex", -1);
////////////////////////////////////////////////////////////////////////
// if running headless, always go straight to the preview screen next //
// todo actually, we could make this execute, right? //
////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput);
}
else
{
//////////////////////////////////////////////////////////////////////////////////
// else - if no values to map - continue with the standard streamed-ETL preview //
//////////////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput);
if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping))
{
//////////////////////////////////////////////////////////////////////////////////
// just go to the prepareValueMapping backend step - it'll figure out the rest. //
// it's also where the value-mapping loop of steps points. //
// and, this will actually be the default (e.g., the step after this one). //
//////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.addValue("valueMappingFieldIndex", -1);
}
else
{
//////////////////////////////////////////////////////////////////////////////////
// else - if no values to map - continue with the standard streamed-ETL preview //
//////////////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput);
}
}
}
catch(Exception e)

39
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java
View File

@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile;
@ -69,6 +70,18 @@ public class BulkInsertStepUtils
/***************************************************************************
**
***************************************************************************/
public static void setStorageInputForTheFile(RunProcessInput runProcessInput, StorageInput storageInput)
{
ArrayList<StorageInput> storageInputs = new ArrayList<>();
storageInputs.add(storageInput);
runProcessInput.addValue("theFile", storageInputs);
}
/***************************************************************************
**
***************************************************************************/
@ -144,13 +157,37 @@ public class BulkInsertStepUtils
/***************************************************************************
**
***************************************************************************/
public static void handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
public static QRecord handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("savedBulkLoadProfileId");
if(savedBulkLoadProfileId != null)
{
QRecord savedBulkLoadProfileRecord = GetAction.execute(SavedBulkLoadProfile.TABLE_NAME, savedBulkLoadProfileId);
runBackendStepOutput.addValue("savedBulkLoadProfileRecord", savedBulkLoadProfileRecord);
return (savedBulkLoadProfileRecord);
}
return (null);
}
/***************************************************************************
**
***************************************************************************/
public static boolean isHeadless(RunBackendStepInput runBackendStepInput)
{
return (runBackendStepInput.getValuePrimitiveBoolean("isHeadless"));
}
/***************************************************************************
**
***************************************************************************/
public static void setHeadless(RunProcessInput runProcessInput)
{
runProcessInput.addValue("isHeadless", true);
}
}