diff --git a/.circleci/config.yml b/.circleci/config.yml index 40a1a443..573814e9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,7 +57,7 @@ jobs: - localstack/startup - install_java17 - run_maven: - maven_subcommand: test + maven_subcommand: verify - slack/notify: event: fail diff --git a/checkstyle.xml b/checkstyle.xml index 76f872ed..f5e7412d 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -181,8 +181,8 @@ --> - @@ -115,6 +117,9 @@ org.apache.maven.plugins maven-surefire-plugin 3.0.0-M5 + + @{jaCoCoArgLine} + org.apache.maven.plugins @@ -164,7 +169,84 @@ 1 - + + org.jacoco + jacoco-maven-plugin + 0.8.8 + + + pre-unit-test + + prepare-agent + + + jaCoCoArgLine + + + + unit-test-check + + check + + + + ${coverage.haltOnFailure} + + + BUNDLE + + + INSTRUCTION + COVEREDRATIO + ${coverage.instructionCoveredRatioMinimum} + + + + + + + + post-unit-test + verify + + report + + + + + + exec-maven-plugin + org.codehaus.mojo + 3.0.0 + + + test-coverage-summary + verify + + exec + + + sh + + -c + + /tmp/$$.headers +xpath -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values +echo +echo "Jacoco coverage summary report:" +echo " See also target/site/jacoco/index.html" +echo " and https://www.jacoco.org/jacoco/trunk/doc/counters.html" +echo "------------------------------------------------------------" +paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}' +rm /tmp/$$.headers /tmp/$$.values + ]]> + + + + + + diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java index 3051e18a..23c38c46 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java @@ -31,6 +31,8 @@ import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFile *******************************************************************************/ public interface FilesystemBackendModuleInterface { + String CUSTOMIZER_FILE_POST_FILE_READ = "postFileRead"; + /******************************************************************************* ** For filesystem backends, get the module-specific action base-class, that helps ** with functions like listing and deleting files. diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 9a04f289..8b211f7f 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; @@ -269,7 +270,7 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ private String customizeFileContentsAfterReading(QTableMetaData table, String fileContents) throws QException { - Optional optionalCustomizer = table.getCustomizer("postFileRead"); + Optional optionalCustomizer = table.getCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ); if(optionalCustomizer.isEmpty()) { return (fileContents); diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java similarity index 72% rename from src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java rename to src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java index f91a5b10..f2d3826b 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java @@ -24,9 +24,9 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et import java.io.File; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.interfaces.BackendStep; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; @@ -34,8 +34,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; @@ -47,65 +47,65 @@ import org.apache.logging.log4j.Logger; /******************************************************************************* - ** Function body for performing the Cleanup step of a basic ETL process - e.g., + ** BackendStep for performing the Cleanup step of a basic ETL process - e.g., ** after the loading, delete or move the processed file(s). *******************************************************************************/ -public class BasicETLCleanupSourceFilesFunction implements FunctionBody +public class BasicETLCleanupSourceFilesStep implements BackendStep { - private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesFunction.class); + private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesStep.class); public static final String FIELD_MOVE_OR_DELETE = "moveOrDelete"; public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves"; public static final String VALUE_MOVE = "move"; - public static final String VALUE_DELETE = "delete"; - public static final String FUNCTION_NAME = "cleanupSourceFiles"; + public static final String VALUE_DELETE = "delete"; + public static final String STEP_NAME = "cleanupSourceFiles"; /******************************************************************************* - ** Execute the function - using the request as input, and the result as output. + ** Execute the step - using the request as input, and the result as output. *******************************************************************************/ @Override - public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException { - String sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE); - QTableMetaData table = runFunctionRequest.getInstance().getTable(sourceTableName); - QBackendMetaData backend = runFunctionRequest.getInstance().getBackendForTable(sourceTableName); + String sourceTableName = runBackendStepRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE); + QTableMetaData table = runBackendStepRequest.getInstance().getTable(sourceTableName); + QBackendMetaData backend = runBackendStepRequest.getInstance().getBackendForTable(sourceTableName); QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend); if(!(module instanceof FilesystemBackendModuleInterface filesystemModule)) { - throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function.")); + throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this action.")); } AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase(); actionBase.preAction(backend); - String sourceFilePaths = runFunctionRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS); + String sourceFilePaths = runBackendStepRequest.getValueString(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS); if(!StringUtils.hasContent(sourceFilePaths)) { - LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS + "]"); + LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS + "]"); return; } String[] sourceFiles = sourceFilePaths.split(","); for(String sourceFile : sourceFiles) { - String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE); + String moveOrDelete = runBackendStepRequest.getValueString(FIELD_MOVE_OR_DELETE); if(VALUE_DELETE.equals(moveOrDelete)) { - actionBase.deleteFile(runFunctionRequest.getInstance(), table, sourceFile); + actionBase.deleteFile(runBackendStepRequest.getInstance(), table, sourceFile); } else if(VALUE_MOVE.equals(moveOrDelete)) { - String destinationForMoves = runFunctionRequest.getValueString(FIELD_DESTINATION_FOR_MOVES); + String destinationForMoves = runBackendStepRequest.getValueString(FIELD_DESTINATION_FOR_MOVES); if(!StringUtils.hasContent(destinationForMoves)) { throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value.")); } String filePathWithoutBase = actionBase.stripBackendAndTableBasePathsFromFileName(sourceFile, backend, table); String destinationPath = destinationForMoves + File.separator + filePathWithoutBase; - actionBase.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath); + actionBase.moveFile(runBackendStepRequest.getInstance(), table, sourceFile, destinationPath); } else { @@ -118,16 +118,16 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody /******************************************************************************* - ** define the metaData that describes this function + ** define the metaData that describes this step *******************************************************************************/ - public QFunctionMetaData defineFunctionMetaData() + public QBackendStepMetaData defineStepMetaData() { - return (new QFunctionMetaData() - .withName(FUNCTION_NAME) + return (new QBackendStepMetaData() + .withName(STEP_NAME) .withCode(new QCodeReference() .withName(this.getClass().getName()) .withCodeType(QCodeType.JAVA) - .withCodeUsage(QCodeUsage.FUNCTION)) + .withCodeUsage(QCodeUsage.BACKEND_STEP)) .withInputData(new QFunctionInputMetaData() .addField(new QFieldMetaData("moveOrDelete", QFieldType.STRING)) .addField(new QFieldMetaData("destinationForMoves", QFieldType.STRING)))); diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java similarity index 75% rename from src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java rename to src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java index 20de530f..d0e8581c 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java @@ -25,59 +25,59 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.interfaces.BackendStep; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult; import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; /******************************************************************************* - ** Function body for collecting the file names that were discovered in the + ** BackendStep for collecting the file names that were discovered in the ** Extract step. These will be lost during the transform, so we capture them here, - ** so that our Clean function can move or delete them. + ** so that our Clean step can move or delete them. ** ** TODO - need unit test!! *******************************************************************************/ -public class BasicETLCollectSourceFileNamesFunction implements FunctionBody +public class BasicETLCollectSourceFileNamesStep implements BackendStep { - public static final String FUNCTION_NAME = "collectSourceFileNames"; + public static final String STEP_NAME = "collectSourceFileNames"; public static final String FIELD_SOURCE_FILE_PATHS = "sourceFilePaths"; /******************************************************************************* - ** Execute the function - using the request as input, and the result as output. + ** Execute the step - using the request as input, and the result as output. *******************************************************************************/ @Override - public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException { - Set sourceFiles = runFunctionRequest.getRecords().stream() + Set sourceFiles = runBackendStepRequest.getRecords().stream() .map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH)) .collect(Collectors.toSet()); - runFunctionResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles)); + runBackendStepResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles)); } /******************************************************************************* - ** define the metaData that describes this function + ** define the metaData that describes this step *******************************************************************************/ - public QFunctionMetaData defineFunctionMetaData() + public QBackendStepMetaData defineStepMetaData() { - return (new QFunctionMetaData() - .withName(FUNCTION_NAME) + return (new QBackendStepMetaData() + .withName(STEP_NAME) .withCode(new QCodeReference() .withName(this.getClass().getName()) .withCodeType(QCodeType.JAVA) - .withCodeUsage(QCodeUsage.FUNCTION)) + .withCodeUsage(QCodeUsage.BACKEND_STEP)) .withOutputMetaData(new QFunctionOutputMetaData() .addField(new QFieldMetaData(FIELD_SOURCE_FILE_PATHS, QFieldType.STRING)))); } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java index 5773630a..aca2347a 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java @@ -27,8 +27,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; @@ -64,12 +64,12 @@ public class FilesystemSyncProcess *******************************************************************************/ public QProcessMetaData defineProcessMetaData() { - QFunctionMetaData syncFunction = new QFunctionMetaData() - .withName(FilesystemSyncFunction.FUNCTION_NAME) + QBackendStepMetaData syncStep = new QBackendStepMetaData() + .withName(FilesystemSyncStep.STEP_NAME) .withCode(new QCodeReference() - .withName(FilesystemSyncFunction.class.getName()) + .withName(FilesystemSyncStep.class.getName()) .withCodeType(QCodeType.JAVA) - .withCodeUsage(QCodeUsage.FUNCTION)) + .withCodeUsage(QCodeUsage.BACKEND_STEP)) .withInputData(new QFunctionInputMetaData() .addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)) .addField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING)) @@ -78,6 +78,6 @@ public class FilesystemSyncProcess return new QProcessMetaData() .withName(PROCESS_NAME) - .addFunction(syncFunction); + .addStep(syncStep); } } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java similarity index 77% rename from src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java rename to src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java index 45264376..bedf2616 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java @@ -29,9 +29,9 @@ import java.util.List; import java.util.Map; import java.util.Set; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.interfaces.BackendStep; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; @@ -42,47 +42,47 @@ import org.apache.logging.log4j.Logger; /******************************************************************************* - ** Function body for collecting the file names that were discovered in the - ** Extract step. These will be lost during the transform, so we capture them here, - ** so that our Clean function can move or delete them. + ** BackendStep to sync two filesystem tables (copying the new files to a 3rd + ** location as well...) ** *******************************************************************************/ -public class FilesystemSyncFunction implements FunctionBody +@SuppressWarnings("unchecked") +public class FilesystemSyncStep implements BackendStep { - private static final Logger LOG = LogManager.getLogger(FilesystemSyncFunction.class); + private static final Logger LOG = LogManager.getLogger(FilesystemSyncStep.class); - public static final String FUNCTION_NAME = "sync"; + public static final String STEP_NAME = "sync"; /******************************************************************************* - ** Execute the function - using the request as input, and the result as output. + ** Execute the step - using the request as input, and the result as output. *******************************************************************************/ @Override - public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException { - QTableMetaData sourceTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE)); - QTableMetaData archiveTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE)); - QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE)); + QTableMetaData sourceTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE)); + QTableMetaData archiveTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE)); + QTableMetaData processingTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE)); - QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName()); + QBackendMetaData sourceBackend = runBackendStepRequest.getInstance().getBackendForTable(sourceTable.getName()); FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend); AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase(); sourceActionBase.preAction(sourceBackend); Map sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend); - QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName()); + QBackendMetaData archiveBackend = runBackendStepRequest.getInstance().getBackendForTable(archiveTable.getName()); FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend); AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase(); archiveActionBase.preAction(archiveBackend); Set archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet(); - QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName()); + QBackendMetaData processingBackend = runBackendStepRequest.getInstance().getBackendForTable(processingTable.getName()); FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend); AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase(); processingActionBase.preAction(processingBackend); - Integer maxFilesToSync = runFunctionRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE); + Integer maxFilesToSync = runBackendStepRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE); int syncedFileCount = 0; for(Map.Entry sourceEntry : sourceFiles.entrySet()) { diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 21958ab7..b7fc6ca7 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -27,6 +27,7 @@ import java.io.InputStream; import java.util.List; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -67,6 +68,17 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)), @@ -57,12 +65,36 @@ public class FilesystemQueryActionTest extends FilesystemActionTest /******************************************************************************* ** *******************************************************************************/ - private QueryRequest initQueryRequest() throws QInstanceValidationException + @Test + public void testQueryWithFileCustomizer() throws QException { QueryRequest queryRequest = new QueryRequest(); - queryRequest.setInstance(TestUtils.defineInstance()); + QInstance instance = TestUtils.defineInstance(); + + QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + table.withCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ, new QCodeReference() + .withName(ValueUpshifter.class.getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.CUSTOMIZER)); + + queryRequest.setInstance(instance); queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName()); - return queryRequest; + QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest); + Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows"); + Assertions.assertTrue( + queryResult.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")), + "All records should have their email addresses up-shifted."); + } + + + + public static class ValueUpshifter implements Function + { + @Override + public String apply(String s) + { + return (s.replaceAll("kingsrook.com", "KINGSROOK.COM")); + } } } \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java similarity index 79% rename from src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java rename to src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java index 8da9dbda..e3a508d4 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java @@ -27,11 +27,11 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.UUID; -import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -48,7 +48,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* ** Unit test for BasicETLCleanupSourceFilesFunction *******************************************************************************/ -public class BasicETLCleanupSourceFilesFunctionTest +public class BasicETLCleanupSourceFilesStepTest { /******************************************************************************* @@ -136,13 +136,13 @@ public class BasicETLCleanupSourceFilesFunctionTest *******************************************************************************/ private void testDelete(QInstance qInstance, List filePaths) throws Exception { - RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of( - BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_DELETE, + RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of( + BasicETLCleanupSourceFilesStep.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesStep.VALUE_DELETE, // todo - even though this field isn't needed, since we gave a value of "delete" // the RunFunctionAction considers any missing input to be an error... - BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, "")); + BasicETLCleanupSourceFilesStep.FIELD_DESTINATION_FOR_MOVES, "")); - assertNull(runFunctionResult.getError()); + assertNull(runBackendStepResult.getException()); for(String filePath : filePaths) { assertFalse(new File(filePath).exists(), "File should have been deleted."); @@ -157,11 +157,11 @@ public class BasicETLCleanupSourceFilesFunctionTest private void testMove(QInstance qInstance, List filePaths) throws Exception { String trashDir = File.separator + "tmp" + File.separator + "trash"; - RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of( - BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_MOVE, - BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir)); + RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of( + BasicETLCleanupSourceFilesStep.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesStep.VALUE_MOVE, + BasicETLCleanupSourceFilesStep.FIELD_DESTINATION_FOR_MOVES, trashDir)); - assertNull(runFunctionResult.getError()); + assertNull(runBackendStepResult.getException()); for(String filePath : filePaths) { @@ -177,10 +177,10 @@ public class BasicETLCleanupSourceFilesFunctionTest /******************************************************************************* ** *******************************************************************************/ - private RunFunctionResult runFunction(QInstance qInstance, List filePaths, Map values) throws Exception + private RunBackendStepResult runFunction(QInstance qInstance, List filePaths, Map values) throws Exception { - QFunctionMetaData qFunctionMetaData = new BasicETLCleanupSourceFilesFunction().defineFunctionMetaData(); - QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addFunction(qFunctionMetaData); + QBackendStepMetaData backendStepMetaData = new BasicETLCleanupSourceFilesStep().defineStepMetaData(); + QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData); qInstance.addProcess(qProcessMetaData); HashSet filePathsSet = new HashSet<>(filePaths); @@ -193,22 +193,22 @@ public class BasicETLCleanupSourceFilesFunctionTest // List records = filePaths.stream() // .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList(); - RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); - runFunctionRequest.setFunctionName(qFunctionMetaData.getName()); - runFunctionRequest.setProcessName(qProcessMetaData.getName()); + RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance); + runBackendStepRequest.setStepName(backendStepMetaData.getName()); + runBackendStepRequest.setProcessName(qProcessMetaData.getName()); // runFunctionRequest.setRecords(records); - runFunctionRequest.setSession(TestUtils.getMockSession()); - runFunctionRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS); - runFunctionRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3); - runFunctionRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet)); + runBackendStepRequest.setSession(TestUtils.getMockSession()); + runBackendStepRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + runBackendStepRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3); + runBackendStepRequest.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet)); for(Map.Entry entry : values.entrySet()) { - runFunctionRequest.addValue(entry.getKey(), entry.getValue()); + runBackendStepRequest.addValue(entry.getKey(), entry.getValue()); } - RunFunctionAction runFunctionAction = new RunFunctionAction(); - return (runFunctionAction.execute(runFunctionRequest)); + RunBackendStepAction runFunctionAction = new RunBackendStepAction(); + return (runFunctionAction.execute(runBackendStepRequest)); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStepTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStepTest.java new file mode 100644 index 00000000..db56b418 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStepTest.java @@ -0,0 +1,115 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic; + + +import java.util.Arrays; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for BasicETLCollectSourceFileNamesFunction + *******************************************************************************/ +class BasicETLCollectSourceFileNamesStepTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testOneFile() throws Exception + { + String file = "/tmp/test1.csv"; + String result = runTest(file); + assertEquals(file, result); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testTwoFiles() throws Exception + { + String file1 = "/tmp/test1.csv"; + String file2 = "/tmp/test2.csv"; + String result = runTest(file1, file2); + + ////////////////////////////////////////////////////////////////////// + // the names go into a set, so they can come out in either order... // + ////////////////////////////////////////////////////////////////////// + assertTrue(result.equals(file1 + "," + file2) || result.equals((file2 + "," + file1))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDuplicatedFile() throws Exception + { + String file = "/tmp/test1.csv"; + String result = runTest(file, file); + assertEquals(file, result); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String runTest(String... fileNames) throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QBackendStepMetaData backendStepMetaData = new BasicETLCollectSourceFileNamesStep().defineStepMetaData(); + QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData); + qInstance.addProcess(qProcessMetaData); + + List records = Arrays.stream(fileNames).map(fileName -> + new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fileName)).toList(); + + RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance); + runBackendStepRequest.setSession(TestUtils.getMockSession()); + runBackendStepRequest.setStepName(backendStepMetaData.getName()); + runBackendStepRequest.setProcessName(qProcessMetaData.getName()); + runBackendStepRequest.setRecords(records); + + RunBackendStepAction runBackendStepAction = new RunBackendStepAction(); + RunBackendStepResult result = runBackendStepAction.execute(runBackendStepRequest); + + return ((String) result.getValues().get(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS)); + } +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java index 2064617c..4b3d87b7 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java @@ -25,16 +25,16 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi import java.util.List; import java.util.stream.Collectors; import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; +import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction; import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; @@ -80,13 +80,13 @@ class FilesystemSyncProcessS3Test extends BaseS3Test QTableMetaData archiveTable = defineTable(qInstance, "archive", archiveBackend, "archive", "*/l3/*.csv"); QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv"); - QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); - QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); + QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); + QBackendStepMetaData step = process.getBackendStep(FilesystemSyncStep.STEP_NAME); qInstance.addProcess(process); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); /////////////////////////// // write some test files // @@ -99,17 +99,17 @@ class FilesystemSyncProcessS3Test extends BaseS3Test printTableListing(archiveBackend, archiveTable); printTableListing(processingBackend, processingTable); - ////////////////////// - // run the function // - ////////////////////// - RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); - runFunctionRequest.setFunctionName(function.getName()); - runFunctionRequest.setProcessName(process.getName()); - runFunctionRequest.setSession(TestUtils.getMockSession()); + ////////////////// + // run the step // + ////////////////// + RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance); + runBackendStepRequest.setStepName(step.getName()); + runBackendStepRequest.setProcessName(process.getName()); + runBackendStepRequest.setSession(TestUtils.getMockSession()); - RunFunctionAction runFunctionAction = new RunFunctionAction(); - RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); - System.out.println(runFunctionResult); + RunBackendStepAction runFunctionAction = new RunBackendStepAction(); + RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest); + // System.out.println(runBackendStepResult); printTableListing(sourceBackend, sourceTable); printTableListing(archiveBackend, archiveTable); @@ -143,13 +143,13 @@ class FilesystemSyncProcessS3Test extends BaseS3Test QTableMetaData archiveTable = defineTable(qInstance, "archive", localBackend, "archive", "*/l3/*.csv"); QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv"); - QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); - QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); + QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); + QBackendStepMetaData step = process.getBackendStep(FilesystemSyncStep.STEP_NAME); qInstance.addProcess(process); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); /////////////////////////// // write some test files // @@ -162,17 +162,17 @@ class FilesystemSyncProcessS3Test extends BaseS3Test printTableListing(localBackend, archiveTable); printTableListing(localBackend, processingTable); - ////////////////////// - // run the function // - ////////////////////// - RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); - runFunctionRequest.setFunctionName(function.getName()); - runFunctionRequest.setProcessName(process.getName()); - runFunctionRequest.setSession(TestUtils.getMockSession()); + ////////////////// + // run the step // + ////////////////// + RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance); + runBackendStepRequest.setStepName(step.getName()); + runBackendStepRequest.setProcessName(process.getName()); + runBackendStepRequest.setSession(TestUtils.getMockSession()); - RunFunctionAction runFunctionAction = new RunFunctionAction(); - RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); - System.out.println(runFunctionResult); + RunBackendStepAction runFunctionAction = new RunBackendStepAction(); + RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest); + // System.out.println(runBackendStepResult); printTableListing(vendorBackend, sourceTable); printTableListing(localBackend, archiveTable); diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java index 188ef71a..9f0e3933 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java @@ -24,14 +24,14 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi import java.io.File; import java.io.IOException; -import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; @@ -54,16 +54,16 @@ class FilesystemSyncProcessTest { TestUtils.cleanInstanceFiles(); - QTableMetaData sourceTable = defineTable("source"); - QTableMetaData archiveTable = defineTable("archive"); - QTableMetaData processingTable = defineTable("processing"); - QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); - QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); + QTableMetaData sourceTable = defineTable("source"); + QTableMetaData archiveTable = defineTable("archive"); + QTableMetaData processingTable = defineTable("processing"); + QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); + QBackendStepMetaData step = (QBackendStepMetaData) process.getStep(FilesystemSyncStep.STEP_NAME); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); - function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); - // function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); + step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); + // step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1); QInstance qInstance = TestUtils.defineInstance(); qInstance.addTable(sourceTable); @@ -81,16 +81,16 @@ class FilesystemSyncProcessTest writeTestFile(basePath, archiveTable, "2.txt", "x"); ////////////////////// - // run the function // + // run the step // ////////////////////// - RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); - runFunctionRequest.setFunctionName(function.getName()); - runFunctionRequest.setProcessName(process.getName()); - runFunctionRequest.setSession(TestUtils.getMockSession()); + RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance); + runBackendStepRequest.setStepName(step.getName()); + runBackendStepRequest.setProcessName(process.getName()); + runBackendStepRequest.setSession(TestUtils.getMockSession()); - RunFunctionAction runFunctionAction = new RunFunctionAction(); - RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); - System.out.println(runFunctionResult); + RunBackendStepAction runFunctionAction = new RunBackendStepAction(); + RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest); + // System.out.println(runBackendStepResult); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3ActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3ActionTest.java new file mode 100644 index 00000000..5ffe69b6 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3ActionTest.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + + +/******************************************************************************* + ** Unit test for AbstractS3Action + *******************************************************************************/ +class AbstractS3ActionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBuildAmazonS3ClientFromBackendMetaData() + { + String regionName = Regions.AP_SOUTHEAST_3.getName(); + S3BackendMetaData s3BackendMetaData = new S3BackendMetaData() + .withAccessKey("Not a real access key") + .withSecretKey("Also not a real key") + .withRegion(regionName); + AmazonS3 amazonS3 = new AbstractS3Action().buildAmazonS3ClientFromBackendMetaData(s3BackendMetaData); + assertEquals(regionName, amazonS3.getRegionName()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBuildAmazonS3ClientFromBackendMetaDataWrongType() + { + assertThrows(IllegalArgumentException.class, () -> + { + new AbstractS3Action().buildAmazonS3ClientFromBackendMetaData(new QBackendMetaData()); + }); + } +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java index acef8ec6..58b57634 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java @@ -52,7 +52,7 @@ class S3BackendMetaDataTest System.out.println(JsonUtils.prettyPrint(json)); System.out.println(json); String expectToContain = """ - {"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","secretKey":null,"accessKey":null,"backendType":"s3","name":"s3","region":null}"""; + {"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","backendType":"s3","name":"s3"}"""; assertTrue(json.contains(expectToContain)); }