diff --git a/pom.xml b/pom.xml index 58f134e9..1d51bc31 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ com.kingsrook.qqq qqq-backend-core - 0.0.0 + 0.1.0-20220706.184937-2 diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java index f91a5b10..d95c4de5 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java @@ -24,9 +24,9 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et import java.io.File; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.interfaces.BackendStep; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; @@ -34,8 +34,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; @@ -50,7 +50,7 @@ import org.apache.logging.log4j.Logger; ** Function body for performing the Cleanup step of a basic ETL process - e.g., ** after the loading, delete or move the processed file(s). *******************************************************************************/ -public class BasicETLCleanupSourceFilesFunction implements FunctionBody +public class BasicETLCleanupSourceFilesFunction implements BackendStep { private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesFunction.class); @@ -67,11 +67,11 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody ** Execute the function - using the request as input, and the result as output. *******************************************************************************/ @Override - public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException { - String sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE); - QTableMetaData table = runFunctionRequest.getInstance().getTable(sourceTableName); - QBackendMetaData backend = runFunctionRequest.getInstance().getBackendForTable(sourceTableName); + String sourceTableName = runBackendStepRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE); + QTableMetaData table = runBackendStepRequest.getInstance().getTable(sourceTableName); + QBackendMetaData backend = runBackendStepRequest.getInstance().getBackendForTable(sourceTableName); QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend); if(!(module instanceof FilesystemBackendModuleInterface filesystemModule)) @@ -81,7 +81,7 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase(); actionBase.preAction(backend); - String sourceFilePaths = runFunctionRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS); + String sourceFilePaths = runBackendStepRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS); if(!StringUtils.hasContent(sourceFilePaths)) { LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS + "]"); @@ -91,21 +91,21 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody String[] sourceFiles = sourceFilePaths.split(","); for(String sourceFile : sourceFiles) { - String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE); + String moveOrDelete = runBackendStepRequest.getValueString(FIELD_MOVE_OR_DELETE); if(VALUE_DELETE.equals(moveOrDelete)) { - actionBase.deleteFile(runFunctionRequest.getInstance(), table, sourceFile); + actionBase.deleteFile(runBackendStepRequest.getInstance(), table, sourceFile); } else if(VALUE_MOVE.equals(moveOrDelete)) { - String destinationForMoves = runFunctionRequest.getValueString(FIELD_DESTINATION_FOR_MOVES); + String destinationForMoves = runBackendStepRequest.getValueString(FIELD_DESTINATION_FOR_MOVES); if(!StringUtils.hasContent(destinationForMoves)) { throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value.")); } String filePathWithoutBase = actionBase.stripBackendAndTableBasePathsFromFileName(sourceFile, backend, table); String destinationPath = destinationForMoves + File.separator + filePathWithoutBase; - actionBase.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath); + actionBase.moveFile(runBackendStepRequest.getInstance(), table, sourceFile, destinationPath); } else { @@ -120,9 +120,9 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody /******************************************************************************* ** define the metaData that describes this function *******************************************************************************/ - public QFunctionMetaData defineFunctionMetaData() + public QBackendStepMetaData defineStepMetaData() { - return (new QFunctionMetaData() + return (new QBackendStepMetaData() .withName(FUNCTION_NAME) .withCode(new QCodeReference() .withName(this.getClass().getName()) diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java index 20de530f..3fa1cbc9 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java @@ -25,15 +25,15 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.interfaces.BackendStep; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult; import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; @@ -46,7 +46,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendD ** ** TODO - need unit test!! *******************************************************************************/ -public class BasicETLCollectSourceFileNamesFunction implements FunctionBody +public class BasicETLCollectSourceFileNamesFunction implements BackendStep { public static final String FUNCTION_NAME = "collectSourceFileNames"; public static final String FIELD_SOURCE_FILE_PATHS = "sourceFilePaths"; @@ -57,12 +57,12 @@ public class BasicETLCollectSourceFileNamesFunction implements FunctionBody ** Execute the function - using the request as input, and the result as output. *******************************************************************************/ @Override - public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException { - Set sourceFiles = runFunctionRequest.getRecords().stream() + Set sourceFiles = runBackendStepRequest.getRecords().stream() .map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH)) .collect(Collectors.toSet()); - runFunctionResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles)); + runBackendStepResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles)); } @@ -70,9 +70,9 @@ public class BasicETLCollectSourceFileNamesFunction implements FunctionBody /******************************************************************************* ** define the metaData that describes this function *******************************************************************************/ - public QFunctionMetaData defineFunctionMetaData() + public QBackendStepMetaData defineStepMetaData() { - return (new QFunctionMetaData() + return (new QBackendStepMetaData() .withName(FUNCTION_NAME) .withCode(new QCodeReference() .withName(this.getClass().getName()) diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java index 45264376..145edf6a 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java @@ -29,9 +29,9 @@ import java.util.List; import java.util.Map; import java.util.Set; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.interfaces.BackendStep; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; @@ -47,7 +47,7 @@ import org.apache.logging.log4j.Logger; ** so that our Clean function can move or delete them. ** *******************************************************************************/ -public class FilesystemSyncFunction implements FunctionBody +public class FilesystemSyncFunction implements BackendStep { private static final Logger LOG = LogManager.getLogger(FilesystemSyncFunction.class); @@ -59,30 +59,30 @@ public class FilesystemSyncFunction implements FunctionBody ** Execute the function - using the request as input, and the result as output. *******************************************************************************/ @Override - public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException { - QTableMetaData sourceTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE)); - QTableMetaData archiveTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE)); - QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE)); + QTableMetaData sourceTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE)); + QTableMetaData archiveTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE)); + QTableMetaData processingTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE)); - QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName()); + QBackendMetaData sourceBackend = runBackendStepRequest.getInstance().getBackendForTable(sourceTable.getName()); FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend); AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase(); sourceActionBase.preAction(sourceBackend); Map sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend); - QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName()); + QBackendMetaData archiveBackend = runBackendStepRequest.getInstance().getBackendForTable(archiveTable.getName()); FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend); AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase(); archiveActionBase.preAction(archiveBackend); Set archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet(); - QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName()); + QBackendMetaData processingBackend = runBackendStepRequest.getInstance().getBackendForTable(processingTable.getName()); FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend); AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase(); processingActionBase.preAction(processingBackend); - Integer maxFilesToSync = runFunctionRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE); + Integer maxFilesToSync = runBackendStepRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE); int syncedFileCount = 0; for(Map.Entry sourceEntry : sourceFiles.entrySet()) { diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java index 5773630a..39a708fb 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java @@ -27,8 +27,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; @@ -64,7 +64,7 @@ public class FilesystemSyncProcess *******************************************************************************/ public QProcessMetaData defineProcessMetaData() { - QFunctionMetaData syncFunction = new QFunctionMetaData() + QBackendStepMetaData syncFunction = new QBackendStepMetaData() .withName(FilesystemSyncFunction.FUNCTION_NAME) .withCode(new QCodeReference() .withName(FilesystemSyncFunction.class.getName()) @@ -78,6 +78,6 @@ public class FilesystemSyncProcess return new QProcessMetaData() .withName(PROCESS_NAME) - .addFunction(syncFunction); + .addStep(syncFunction); } } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java index 8da9dbda..5a4e9fd2 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java @@ -27,11 +27,11 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.UUID; -import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -136,13 +136,13 @@ public class BasicETLCleanupSourceFilesFunctionTest *******************************************************************************/ private void testDelete(QInstance qInstance, List filePaths) throws Exception { - RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of( + RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of( BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_DELETE, // todo - even though this field isn't needed, since we gave a value of "delete" // the RunFunctionAction considers any missing input to be an error... BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, "")); - assertNull(runFunctionResult.getError()); + assertNull(runBackendStepResult.getError()); for(String filePath : filePaths) { assertFalse(new File(filePath).exists(), "File should have been deleted."); @@ -157,11 +157,11 @@ public class BasicETLCleanupSourceFilesFunctionTest private void testMove(QInstance qInstance, List filePaths) throws Exception { String trashDir = File.separator + "tmp" + File.separator + "trash"; - RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of( + RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of( BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_MOVE, BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir)); - assertNull(runFunctionResult.getError()); + assertNull(runBackendStepResult.getError()); for(String filePath : filePaths) { @@ -177,10 +177,10 @@ public class BasicETLCleanupSourceFilesFunctionTest /******************************************************************************* ** *******************************************************************************/ - private RunFunctionResult runFunction(QInstance qInstance, List filePaths, Map values) throws Exception + private RunBackendStepResult runFunction(QInstance qInstance, List filePaths, Map values) throws Exception { - QFunctionMetaData qFunctionMetaData = new BasicETLCleanupSourceFilesFunction().defineFunctionMetaData(); - QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addFunction(qFunctionMetaData); + QBackendStepMetaData backendStepMetaData = new BasicETLCleanupSourceFilesFunction().defineStepMetaData(); + QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData); qInstance.addProcess(qProcessMetaData); HashSet filePathsSet = new HashSet<>(filePaths); @@ -193,22 +193,22 @@ public class BasicETLCleanupSourceFilesFunctionTest // List records = filePaths.stream() // .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList(); - RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); - runFunctionRequest.setFunctionName(qFunctionMetaData.getName()); - runFunctionRequest.setProcessName(qProcessMetaData.getName()); + RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance); + runBackendStepRequest.setStepName(backendStepMetaData.getName()); + runBackendStepRequest.setProcessName(qProcessMetaData.getName()); // runFunctionRequest.setRecords(records); - runFunctionRequest.setSession(TestUtils.getMockSession()); - runFunctionRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS); - runFunctionRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3); - runFunctionRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet)); + runBackendStepRequest.setSession(TestUtils.getMockSession()); + runBackendStepRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + runBackendStepRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3); + runBackendStepRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet)); for(Map.Entry entry : values.entrySet()) { - runFunctionRequest.addValue(entry.getKey(), entry.getValue()); + runBackendStepRequest.addValue(entry.getKey(), entry.getValue()); } - RunFunctionAction runFunctionAction = new RunFunctionAction(); - return (runFunctionAction.execute(runFunctionRequest)); + RunBackendStepAction runFunctionAction = new RunBackendStepAction(); + return (runFunctionAction.execute(runBackendStepRequest)); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java index 2064617c..5fc19bda 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java @@ -25,16 +25,16 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi import java.util.List; import java.util.stream.Collectors; import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; +import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction; import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; @@ -80,13 +80,13 @@ class FilesystemSyncProcessS3Test extends BaseS3Test QTableMetaData archiveTable = defineTable(qInstance, "archive", archiveBackend, "archive", "*/l3/*.csv"); QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv"); - QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); - QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); + QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); + QBackendStepMetaData step = process.getBackendStep(FilesystemSyncFunction.FUNCTION_NAME); qInstance.addProcess(process); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); /////////////////////////// // write some test files // @@ -99,17 +99,17 @@ class FilesystemSyncProcessS3Test extends BaseS3Test printTableListing(archiveBackend, archiveTable); printTableListing(processingBackend, processingTable); - ////////////////////// - // run the function // - ////////////////////// - RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); - runFunctionRequest.setFunctionName(function.getName()); - runFunctionRequest.setProcessName(process.getName()); - runFunctionRequest.setSession(TestUtils.getMockSession()); + ////////////////// + // run the step // + ////////////////// + RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance); + runBackendStepRequest.setStepName(step.getName()); + runBackendStepRequest.setProcessName(process.getName()); + runBackendStepRequest.setSession(TestUtils.getMockSession()); - RunFunctionAction runFunctionAction = new RunFunctionAction(); - RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); - System.out.println(runFunctionResult); + RunBackendStepAction runFunctionAction = new RunBackendStepAction(); + RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest); + System.out.println(runBackendStepResult); printTableListing(sourceBackend, sourceTable); printTableListing(archiveBackend, archiveTable); @@ -143,13 +143,13 @@ class FilesystemSyncProcessS3Test extends BaseS3Test QTableMetaData archiveTable = defineTable(qInstance, "archive", localBackend, "archive", "*/l3/*.csv"); QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv"); - QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); - QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); + QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); + QBackendStepMetaData step = process.getBackendStep(FilesystemSyncFunction.FUNCTION_NAME); qInstance.addProcess(process); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); /////////////////////////// // write some test files // @@ -162,17 +162,17 @@ class FilesystemSyncProcessS3Test extends BaseS3Test printTableListing(localBackend, archiveTable); printTableListing(localBackend, processingTable); - ////////////////////// - // run the function // - ////////////////////// - RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); - runFunctionRequest.setFunctionName(function.getName()); - runFunctionRequest.setProcessName(process.getName()); - runFunctionRequest.setSession(TestUtils.getMockSession()); + ////////////////// + // run the step // + ////////////////// + RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance); + runBackendStepRequest.setStepName(step.getName()); + runBackendStepRequest.setProcessName(process.getName()); + runBackendStepRequest.setSession(TestUtils.getMockSession()); - RunFunctionAction runFunctionAction = new RunFunctionAction(); - RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); - System.out.println(runFunctionResult); + RunBackendStepAction runFunctionAction = new RunBackendStepAction(); + RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest); + System.out.println(runBackendStepResult); printTableListing(vendorBackend, sourceTable); printTableListing(localBackend, archiveTable); diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java index 188ef71a..2bcb5158 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java @@ -24,14 +24,14 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi import java.io.File; import java.io.IOException; -import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; @@ -54,16 +54,16 @@ class FilesystemSyncProcessTest { TestUtils.cleanInstanceFiles(); - QTableMetaData sourceTable = defineTable("source"); - QTableMetaData archiveTable = defineTable("archive"); - QTableMetaData processingTable = defineTable("processing"); - QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); - QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); + QTableMetaData sourceTable = defineTable("source"); + QTableMetaData archiveTable = defineTable("archive"); + QTableMetaData processingTable = defineTable("processing"); + QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); + QBackendStepMetaData step = (QBackendStepMetaData) process.getStep(FilesystemSyncFunction.FUNCTION_NAME); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); - // function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); + // step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1); QInstance qInstance = TestUtils.defineInstance(); qInstance.addTable(sourceTable); @@ -81,16 +81,16 @@ class FilesystemSyncProcessTest writeTestFile(basePath, archiveTable, "2.txt", "x"); ////////////////////// - // run the function // + // run the step // ////////////////////// - RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); - runFunctionRequest.setFunctionName(function.getName()); - runFunctionRequest.setProcessName(process.getName()); - runFunctionRequest.setSession(TestUtils.getMockSession()); + RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance); + runBackendStepRequest.setStepName(step.getName()); + runBackendStepRequest.setProcessName(process.getName()); + runBackendStepRequest.setSession(TestUtils.getMockSession()); - RunFunctionAction runFunctionAction = new RunFunctionAction(); - RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); - System.out.println(runFunctionResult); + RunBackendStepAction runFunctionAction = new RunBackendStepAction(); + RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest); + System.out.println(runBackendStepResult); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java index acef8ec6..58b57634 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java @@ -52,7 +52,7 @@ class S3BackendMetaDataTest System.out.println(JsonUtils.prettyPrint(json)); System.out.println(json); String expectToContain = """ - {"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","secretKey":null,"accessKey":null,"backendType":"s3","name":"s3","region":null}"""; + {"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","backendType":"s3","name":"s3"}"""; assertTrue(json.contains(expectToContain)); }