QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

QQQ-21 updated per updated backend-core - process function -> step renames

Browse Source
This commit is contained in:
darin.kelkhoff
2022-07-06 13:53:33 -05:00
parent 2deaf9cc8e
commit 1252bf5af2
9 changed files with 119 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -51,7 +51,7 @@
<dependency> <dependency>
<groupId>com.kingsrook.qqq</groupId> <groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-backend-core</artifactId> <artifactId>qqq-backend-core</artifactId>
<version>0.0.0</version> <version>0.1.0-20220706.184937-2</version>
</dependency> </dependency>
<!-- 3rd party deps specifically for this module --> <!-- 3rd party deps specifically for this module -->

32
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java
View File

@ -24,9 +24,9 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et
import java.io.File; import java.io.File;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
@ -34,8 +34,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
@ -50,7 +50,7 @@ import org.apache.logging.log4j.Logger;
** Function body for performing the Cleanup step of a basic ETL process - e.g., ** Function body for performing the Cleanup step of a basic ETL process - e.g.,
** after the loading, delete or move the processed file(s). ** after the loading, delete or move the processed file(s).
*******************************************************************************/ *******************************************************************************/
public class BasicETLCleanupSourceFilesFunction implements FunctionBody public class BasicETLCleanupSourceFilesFunction implements BackendStep
{ {
private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesFunction.class); private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesFunction.class);
@ -67,11 +67,11 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
** Execute the function - using the request as input, and the result as output. ** Execute the function - using the request as input, and the result as output.
*******************************************************************************/ *******************************************************************************/
@Override @Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
{ {
String sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE); String sourceTableName = runBackendStepRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
QTableMetaData table = runFunctionRequest.getInstance().getTable(sourceTableName); QTableMetaData table = runBackendStepRequest.getInstance().getTable(sourceTableName);
QBackendMetaData backend = runFunctionRequest.getInstance().getBackendForTable(sourceTableName); QBackendMetaData backend = runBackendStepRequest.getInstance().getBackendForTable(sourceTableName);
QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend); QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend);
if(!(module instanceof FilesystemBackendModuleInterface filesystemModule)) if(!(module instanceof FilesystemBackendModuleInterface filesystemModule))
@ -81,7 +81,7 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase(); AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase();
actionBase.preAction(backend); actionBase.preAction(backend);
String sourceFilePaths = runFunctionRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS); String sourceFilePaths = runBackendStepRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS);
if(!StringUtils.hasContent(sourceFilePaths)) if(!StringUtils.hasContent(sourceFilePaths))
{ {
LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS + "]"); LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS + "]");
@ -91,21 +91,21 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
String[] sourceFiles = sourceFilePaths.split(","); String[] sourceFiles = sourceFilePaths.split(",");
for(String sourceFile : sourceFiles) for(String sourceFile : sourceFiles)
{ {
String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE); String moveOrDelete = runBackendStepRequest.getValueString(FIELD_MOVE_OR_DELETE);
if(VALUE_DELETE.equals(moveOrDelete)) if(VALUE_DELETE.equals(moveOrDelete))
{ {
actionBase.deleteFile(runFunctionRequest.getInstance(), table, sourceFile); actionBase.deleteFile(runBackendStepRequest.getInstance(), table, sourceFile);
} }
else if(VALUE_MOVE.equals(moveOrDelete)) else if(VALUE_MOVE.equals(moveOrDelete))
{ {
String destinationForMoves = runFunctionRequest.getValueString(FIELD_DESTINATION_FOR_MOVES); String destinationForMoves = runBackendStepRequest.getValueString(FIELD_DESTINATION_FOR_MOVES);
if(!StringUtils.hasContent(destinationForMoves)) if(!StringUtils.hasContent(destinationForMoves))
{ {
throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value.")); throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value."));
} }
String filePathWithoutBase = actionBase.stripBackendAndTableBasePathsFromFileName(sourceFile, backend, table); String filePathWithoutBase = actionBase.stripBackendAndTableBasePathsFromFileName(sourceFile, backend, table);
String destinationPath = destinationForMoves + File.separator + filePathWithoutBase; String destinationPath = destinationForMoves + File.separator + filePathWithoutBase;
actionBase.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath); actionBase.moveFile(runBackendStepRequest.getInstance(), table, sourceFile, destinationPath);
} }
else else
{ {
@ -120,9 +120,9 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
/******************************************************************************* /*******************************************************************************
** define the metaData that describes this function ** define the metaData that describes this function
*******************************************************************************/ *******************************************************************************/
public QFunctionMetaData defineFunctionMetaData() public QBackendStepMetaData defineStepMetaData()
{ {
return (new QFunctionMetaData() return (new QBackendStepMetaData()
.withName(FUNCTION_NAME) .withName(FUNCTION_NAME)
.withCode(new QCodeReference() .withCode(new QCodeReference()
.withName(this.getClass().getName()) .withName(this.getClass().getName())

20
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java
View File

@ -25,15 +25,15 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
@ -46,7 +46,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendD
** **
** TODO - need unit test!! ** TODO - need unit test!!
*******************************************************************************/ *******************************************************************************/
public class BasicETLCollectSourceFileNamesFunction implements FunctionBody public class BasicETLCollectSourceFileNamesFunction implements BackendStep
{ {
public static final String FUNCTION_NAME = "collectSourceFileNames"; public static final String FUNCTION_NAME = "collectSourceFileNames";
public static final String FIELD_SOURCE_FILE_PATHS = "sourceFilePaths"; public static final String FIELD_SOURCE_FILE_PATHS = "sourceFilePaths";
@ -57,12 +57,12 @@ public class BasicETLCollectSourceFileNamesFunction implements FunctionBody
** Execute the function - using the request as input, and the result as output. ** Execute the function - using the request as input, and the result as output.
*******************************************************************************/ *******************************************************************************/
@Override @Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
{ {
Set<String> sourceFiles = runFunctionRequest.getRecords().stream() Set<String> sourceFiles = runBackendStepRequest.getRecords().stream()
.map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH)) .map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))
.collect(Collectors.toSet()); .collect(Collectors.toSet());
runFunctionResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles)); runBackendStepResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles));
} }
@ -70,9 +70,9 @@ public class BasicETLCollectSourceFileNamesFunction implements FunctionBody
/******************************************************************************* /*******************************************************************************
** define the metaData that describes this function ** define the metaData that describes this function
*******************************************************************************/ *******************************************************************************/
public QFunctionMetaData defineFunctionMetaData() public QBackendStepMetaData defineStepMetaData()
{ {
return (new QFunctionMetaData() return (new QBackendStepMetaData()
.withName(FUNCTION_NAME) .withName(FUNCTION_NAME)
.withCode(new QCodeReference() .withCode(new QCodeReference()
.withName(this.getClass().getName()) .withName(this.getClass().getName())

24
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java
View File

@ -29,9 +29,9 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
@ -47,7 +47,7 @@ import org.apache.logging.log4j.Logger;
** so that our Clean function can move or delete them. ** so that our Clean function can move or delete them.
** **
*******************************************************************************/ *******************************************************************************/
public class FilesystemSyncFunction implements FunctionBody public class FilesystemSyncFunction implements BackendStep
{ {
private static final Logger LOG = LogManager.getLogger(FilesystemSyncFunction.class); private static final Logger LOG = LogManager.getLogger(FilesystemSyncFunction.class);
@ -59,30 +59,30 @@ public class FilesystemSyncFunction implements FunctionBody
** Execute the function - using the request as input, and the result as output. ** Execute the function - using the request as input, and the result as output.
*******************************************************************************/ *******************************************************************************/
@Override @Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
{ {
QTableMetaData sourceTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE)); QTableMetaData sourceTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
QTableMetaData archiveTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE)); QTableMetaData archiveTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE)); QTableMetaData processingTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName()); QBackendMetaData sourceBackend = runBackendStepRequest.getInstance().getBackendForTable(sourceTable.getName());
FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend); FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase(); AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase();
sourceActionBase.preAction(sourceBackend); sourceActionBase.preAction(sourceBackend);
Map<String, Object> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend); Map<String, Object> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName()); QBackendMetaData archiveBackend = runBackendStepRequest.getInstance().getBackendForTable(archiveTable.getName());
FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend); FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase(); AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase();
archiveActionBase.preAction(archiveBackend); archiveActionBase.preAction(archiveBackend);
Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet(); Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet();
QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName()); QBackendMetaData processingBackend = runBackendStepRequest.getInstance().getBackendForTable(processingTable.getName());
FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend); FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase(); AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase();
processingActionBase.preAction(processingBackend); processingActionBase.preAction(processingBackend);
Integer maxFilesToSync = runFunctionRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE); Integer maxFilesToSync = runBackendStepRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
int syncedFileCount = 0; int syncedFileCount = 0;
for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet()) for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet())
{ {

6
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java
View File

@ -27,8 +27,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
@ -64,7 +64,7 @@ public class FilesystemSyncProcess
*******************************************************************************/ *******************************************************************************/
public QProcessMetaData defineProcessMetaData() public QProcessMetaData defineProcessMetaData()
{ {
QFunctionMetaData syncFunction = new QFunctionMetaData() QBackendStepMetaData syncFunction = new QBackendStepMetaData()
.withName(FilesystemSyncFunction.FUNCTION_NAME) .withName(FilesystemSyncFunction.FUNCTION_NAME)
.withCode(new QCodeReference() .withCode(new QCodeReference()
.withName(FilesystemSyncFunction.class.getName()) .withName(FilesystemSyncFunction.class.getName())
@ -78,6 +78,6 @@ public class FilesystemSyncProcess
return new QProcessMetaData() return new QProcessMetaData()
.withName(PROCESS_NAME) .withName(PROCESS_NAME)
.addFunction(syncFunction); .addStep(syncFunction);
} }
} }

42
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java
View File

@ -27,11 +27,11 @@ import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.UUID; import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -136,13 +136,13 @@ public class BasicETLCleanupSourceFilesFunctionTest
*******************************************************************************/ *******************************************************************************/
private void testDelete(QInstance qInstance, List<String> filePaths) throws Exception private void testDelete(QInstance qInstance, List<String> filePaths) throws Exception
{ {
RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of( RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_DELETE, BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_DELETE,
// todo - even though this field isn't needed, since we gave a value of "delete" // todo - even though this field isn't needed, since we gave a value of "delete"
// the RunFunctionAction considers any missing input to be an error... // the RunFunctionAction considers any missing input to be an error...
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, "")); BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, ""));
assertNull(runFunctionResult.getError()); assertNull(runBackendStepResult.getError());
for(String filePath : filePaths) for(String filePath : filePaths)
{ {
assertFalse(new File(filePath).exists(), "File should have been deleted."); assertFalse(new File(filePath).exists(), "File should have been deleted.");
@ -157,11 +157,11 @@ public class BasicETLCleanupSourceFilesFunctionTest
private void testMove(QInstance qInstance, List<String> filePaths) throws Exception private void testMove(QInstance qInstance, List<String> filePaths) throws Exception
{ {
String trashDir = File.separator + "tmp" + File.separator + "trash"; String trashDir = File.separator + "tmp" + File.separator + "trash";
RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of( RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_MOVE, BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_MOVE,
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir)); BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir));
assertNull(runFunctionResult.getError()); assertNull(runBackendStepResult.getError());
for(String filePath : filePaths) for(String filePath : filePaths)
{ {
@ -177,10 +177,10 @@ public class BasicETLCleanupSourceFilesFunctionTest
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/
private RunFunctionResult runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception private RunBackendStepResult runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception
{ {
QFunctionMetaData qFunctionMetaData = new BasicETLCleanupSourceFilesFunction().defineFunctionMetaData(); QBackendStepMetaData backendStepMetaData = new BasicETLCleanupSourceFilesFunction().defineStepMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addFunction(qFunctionMetaData); QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
qInstance.addProcess(qProcessMetaData); qInstance.addProcess(qProcessMetaData);
HashSet<String> filePathsSet = new HashSet<>(filePaths); HashSet<String> filePathsSet = new HashSet<>(filePaths);
@ -193,22 +193,22 @@ public class BasicETLCleanupSourceFilesFunctionTest
// List<QRecord> records = filePaths.stream() // List<QRecord> records = filePaths.stream()
// .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList(); // .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList();
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runFunctionRequest.setFunctionName(qFunctionMetaData.getName()); runBackendStepRequest.setStepName(backendStepMetaData.getName());
runFunctionRequest.setProcessName(qProcessMetaData.getName()); runBackendStepRequest.setProcessName(qProcessMetaData.getName());
// runFunctionRequest.setRecords(records); // runFunctionRequest.setRecords(records);
runFunctionRequest.setSession(TestUtils.getMockSession()); runBackendStepRequest.setSession(TestUtils.getMockSession());
runFunctionRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS); runBackendStepRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
runFunctionRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3); runBackendStepRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
runFunctionRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet)); runBackendStepRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
for(Map.Entry<String, String> entry : values.entrySet()) for(Map.Entry<String, String> entry : values.entrySet())
{ {
runFunctionRequest.addValue(entry.getKey(), entry.getValue()); runBackendStepRequest.addValue(entry.getKey(), entry.getValue());
} }
RunFunctionAction runFunctionAction = new RunFunctionAction(); RunBackendStepAction runFunctionAction = new RunBackendStepAction();
return (runFunctionAction.execute(runFunctionRequest)); return (runFunctionAction.execute(runBackendStepRequest));
} }

68
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java
View File

@ -25,16 +25,16 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException; import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -80,13 +80,13 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
QTableMetaData archiveTable = defineTable(qInstance, "archive", archiveBackend, "archive", "*/l3/*.csv"); QTableMetaData archiveTable = defineTable(qInstance, "archive", archiveBackend, "archive", "*/l3/*.csv");
QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv"); QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); QBackendStepMetaData step = process.getBackendStep(FilesystemSyncFunction.FUNCTION_NAME);
qInstance.addProcess(process); qInstance.addProcess(process);
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
/////////////////////////// ///////////////////////////
// write some test files // // write some test files //
@ -99,17 +99,17 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
printTableListing(archiveBackend, archiveTable); printTableListing(archiveBackend, archiveTable);
printTableListing(processingBackend, processingTable); printTableListing(processingBackend, processingTable);
////////////////////// //////////////////
// run the function // // run the step //
////////////////////// //////////////////
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runFunctionRequest.setFunctionName(function.getName()); runBackendStepRequest.setStepName(step.getName());
runFunctionRequest.setProcessName(process.getName()); runBackendStepRequest.setProcessName(process.getName());
runFunctionRequest.setSession(TestUtils.getMockSession()); runBackendStepRequest.setSession(TestUtils.getMockSession());
RunFunctionAction runFunctionAction = new RunFunctionAction(); RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
System.out.println(runFunctionResult); System.out.println(runBackendStepResult);
printTableListing(sourceBackend, sourceTable); printTableListing(sourceBackend, sourceTable);
printTableListing(archiveBackend, archiveTable); printTableListing(archiveBackend, archiveTable);
@ -143,13 +143,13 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
QTableMetaData archiveTable = defineTable(qInstance, "archive", localBackend, "archive", "*/l3/*.csv"); QTableMetaData archiveTable = defineTable(qInstance, "archive", localBackend, "archive", "*/l3/*.csv");
QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv"); QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); QBackendStepMetaData step = process.getBackendStep(FilesystemSyncFunction.FUNCTION_NAME);
qInstance.addProcess(process); qInstance.addProcess(process);
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
/////////////////////////// ///////////////////////////
// write some test files // // write some test files //
@ -162,17 +162,17 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
printTableListing(localBackend, archiveTable); printTableListing(localBackend, archiveTable);
printTableListing(localBackend, processingTable); printTableListing(localBackend, processingTable);
////////////////////// //////////////////
// run the function // // run the step //
////////////////////// //////////////////
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runFunctionRequest.setFunctionName(function.getName()); runBackendStepRequest.setStepName(step.getName());
runFunctionRequest.setProcessName(process.getName()); runBackendStepRequest.setProcessName(process.getName());
runFunctionRequest.setSession(TestUtils.getMockSession()); runBackendStepRequest.setSession(TestUtils.getMockSession());
RunFunctionAction runFunctionAction = new RunFunctionAction(); RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
System.out.println(runFunctionResult); System.out.println(runBackendStepResult);
printTableListing(vendorBackend, sourceTable); printTableListing(vendorBackend, sourceTable);
printTableListing(localBackend, archiveTable); printTableListing(localBackend, archiveTable);

42
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java
View File

@ -24,14 +24,14 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
@ -54,16 +54,16 @@ class FilesystemSyncProcessTest
{ {
TestUtils.cleanInstanceFiles(); TestUtils.cleanInstanceFiles();
QTableMetaData sourceTable = defineTable("source"); QTableMetaData sourceTable = defineTable("source");
QTableMetaData archiveTable = defineTable("archive"); QTableMetaData archiveTable = defineTable("archive");
QTableMetaData processingTable = defineTable("processing"); QTableMetaData processingTable = defineTable("processing");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); QBackendStepMetaData step = (QBackendStepMetaData) process.getStep(FilesystemSyncFunction.FUNCTION_NAME);
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
// function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1); // step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1);
QInstance qInstance = TestUtils.defineInstance(); QInstance qInstance = TestUtils.defineInstance();
qInstance.addTable(sourceTable); qInstance.addTable(sourceTable);
@ -81,16 +81,16 @@ class FilesystemSyncProcessTest
writeTestFile(basePath, archiveTable, "2.txt", "x"); writeTestFile(basePath, archiveTable, "2.txt", "x");
////////////////////// //////////////////////
// run the function // // run the step //
////////////////////// //////////////////////
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runFunctionRequest.setFunctionName(function.getName()); runBackendStepRequest.setStepName(step.getName());
runFunctionRequest.setProcessName(process.getName()); runBackendStepRequest.setProcessName(process.getName());
runFunctionRequest.setSession(TestUtils.getMockSession()); runBackendStepRequest.setSession(TestUtils.getMockSession());
RunFunctionAction runFunctionAction = new RunFunctionAction(); RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
System.out.println(runFunctionResult); System.out.println(runBackendStepResult);
} }

2
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java
View File

@ -52,7 +52,7 @@ class S3BackendMetaDataTest
System.out.println(JsonUtils.prettyPrint(json)); System.out.println(JsonUtils.prettyPrint(json));
System.out.println(json); System.out.println(json);
String expectToContain = """ String expectToContain = """
{"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","secretKey":null,"accessKey":null,"backendType":"s3","name":"s3","region":null}"""; {"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","backendType":"s3","name":"s3"}""";
assertTrue(json.contains(expectToContain)); assertTrue(json.contains(expectToContain));
} }