QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/ups-etl-fix' into feature/sprint-7-integration

Browse Source
This commit is contained in:
darin.kelkhoff
2022-07-25 08:19:04 -05:00
parent 164cb2a989 80a6748e50
commit 37e86c1bb4
2 changed files with 20 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLExtractFunction.java
View File

@ -29,6 +29,8 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/******************************************************************************* /*******************************************************************************
@ -36,12 +38,22 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
*******************************************************************************/ *******************************************************************************/
public class BasicETLExtractFunction implements BackendStep public class BasicETLExtractFunction implements BackendStep
{ {
private static final Logger LOG = LogManager.getLogger(BasicETLExtractFunction.class);
/*******************************************************************************
**
*******************************************************************************/
@Override @Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{ {
String tableName = runBackendStepInput.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
LOG.info("Start query on table: " + runBackendStepInput.getTableName());
QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance()); QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance());
queryInput.setSession(runBackendStepInput.getSession()); queryInput.setSession(runBackendStepInput.getSession());
queryInput.setTableName(runBackendStepInput.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE)); queryInput.setTableName(tableName);
// queryRequest.setSkip(integerQueryParam(context, "skip")); // queryRequest.setSkip(integerQueryParam(context, "skip"));
// queryRequest.setLimit(integerQueryParam(context, "limit")); // queryRequest.setLimit(integerQueryParam(context, "limit"));
@ -56,5 +68,6 @@ public class BasicETLExtractFunction implements BackendStep
QueryOutput queryOutput = queryAction.execute(queryInput); QueryOutput queryOutput = queryAction.execute(queryInput);
runBackendStepOutput.setRecords(queryOutput.getRecords()); runBackendStepOutput.setRecords(queryOutput.getRecords());
LOG.info("Query on table " + runBackendStepInput.getTableName() + " produced " + queryOutput.getRecords().size() + " records.");
} }
} }

7
src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunction.java
View File

@ -58,17 +58,22 @@ public class BasicETLTransformFunction implements BackendStep
@Override @Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{ {
String tableName = runBackendStepInput.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
LOG.info("Start transform for destination table: " + tableName);
//////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////
// exit early with no-op if no records made it here, or if we don't have a mapping to use // // exit early with no-op if no records made it here, or if we don't have a mapping to use //
//////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////
if(CollectionUtils.nullSafeIsEmpty(runBackendStepInput.getRecords())) if(CollectionUtils.nullSafeIsEmpty(runBackendStepInput.getRecords()))
{ {
LOG.info("Exiting early with no-op for empty input record list.");
return; return;
} }
String mappingJSON = runBackendStepInput.getValueString(BasicETLProcess.FIELD_MAPPING_JSON); String mappingJSON = runBackendStepInput.getValueString(BasicETLProcess.FIELD_MAPPING_JSON);
if(!StringUtils.hasContent(mappingJSON)) if(!StringUtils.hasContent(mappingJSON))
{ {
LOG.info("Exiting early with no-op for empty mappingJSON.");
return; return;
} }
@ -81,7 +86,6 @@ public class BasicETLTransformFunction implements BackendStep
throw (new QException("Mapping was not a Key-based mapping type. Was a : " + mapping.getClass().getName())); throw (new QException("Mapping was not a Key-based mapping type. Was a : " + mapping.getClass().getName()));
} }
String tableName = runBackendStepInput.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
QTableMetaData table = runBackendStepInput.getInstance().getTable(tableName); QTableMetaData table = runBackendStepInput.getInstance().getTable(tableName);
List<QRecord> mappedRecords = applyMapping(runBackendStepInput.getRecords(), table, keyBasedFieldMapping); List<QRecord> mappedRecords = applyMapping(runBackendStepInput.getRecords(), table, keyBasedFieldMapping);
@ -91,6 +95,7 @@ public class BasicETLTransformFunction implements BackendStep
removeNonNumericValuesFromMappedRecords(table, mappedRecords); removeNonNumericValuesFromMappedRecords(table, mappedRecords);
runBackendStepOutput.setRecords(mappedRecords); runBackendStepOutput.setRecords(mappedRecords);
LOG.info("Done transforming " + runBackendStepOutput.getRecords().size() + " records.");
} }