diff --git a/checkstyle.xml b/checkstyle.xml index 76f872ed..f5e7412d 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -181,8 +181,8 @@ --> - diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java similarity index 89% rename from src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java rename to src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java index d95c4de5..f2d3826b 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java @@ -47,24 +47,24 @@ import org.apache.logging.log4j.Logger; /******************************************************************************* - ** Function body for performing the Cleanup step of a basic ETL process - e.g., + ** BackendStep for performing the Cleanup step of a basic ETL process - e.g., ** after the loading, delete or move the processed file(s). *******************************************************************************/ -public class BasicETLCleanupSourceFilesFunction implements BackendStep +public class BasicETLCleanupSourceFilesStep implements BackendStep { - private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesFunction.class); + private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesStep.class); public static final String FIELD_MOVE_OR_DELETE = "moveOrDelete"; public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves"; public static final String VALUE_MOVE = "move"; - public static final String VALUE_DELETE = "delete"; - public static final String FUNCTION_NAME = "cleanupSourceFiles"; + public static final String VALUE_DELETE = "delete"; + public static final String STEP_NAME = "cleanupSourceFiles"; /******************************************************************************* - ** Execute the function - using the request as input, and the result as output. + ** Execute the step - using the request as input, and the result as output. *******************************************************************************/ @Override public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException @@ -76,15 +76,15 @@ public class BasicETLCleanupSourceFilesFunction implements BackendStep if(!(module instanceof FilesystemBackendModuleInterface filesystemModule)) { - throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function.")); + throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this action.")); } AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase(); actionBase.preAction(backend); - String sourceFilePaths = runBackendStepRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS); + String sourceFilePaths = runBackendStepRequest.getValueString(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS); if(!StringUtils.hasContent(sourceFilePaths)) { - LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS + "]"); + LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS + "]"); return; } @@ -118,16 +118,16 @@ public class BasicETLCleanupSourceFilesFunction implements BackendStep /******************************************************************************* - ** define the metaData that describes this function + ** define the metaData that describes this step *******************************************************************************/ public QBackendStepMetaData defineStepMetaData() { return (new QBackendStepMetaData() - .withName(FUNCTION_NAME) + .withName(STEP_NAME) .withCode(new QCodeReference() .withName(this.getClass().getName()) .withCodeType(QCodeType.JAVA) - .withCodeUsage(QCodeUsage.FUNCTION)) + .withCodeUsage(QCodeUsage.BACKEND_STEP)) .withInputData(new QFunctionInputMetaData() .addField(new QFieldMetaData("moveOrDelete", QFieldType.STRING)) .addField(new QFieldMetaData("destinationForMoves", QFieldType.STRING)))); diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java similarity index 87% rename from src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java rename to src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java index 3fa1cbc9..d0e8581c 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java @@ -40,21 +40,21 @@ import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendD /******************************************************************************* - ** Function body for collecting the file names that were discovered in the + ** BackendStep for collecting the file names that were discovered in the ** Extract step. These will be lost during the transform, so we capture them here, - ** so that our Clean function can move or delete them. + ** so that our Clean step can move or delete them. ** ** TODO - need unit test!! *******************************************************************************/ -public class BasicETLCollectSourceFileNamesFunction implements BackendStep +public class BasicETLCollectSourceFileNamesStep implements BackendStep { - public static final String FUNCTION_NAME = "collectSourceFileNames"; + public static final String STEP_NAME = "collectSourceFileNames"; public static final String FIELD_SOURCE_FILE_PATHS = "sourceFilePaths"; /******************************************************************************* - ** Execute the function - using the request as input, and the result as output. + ** Execute the step - using the request as input, and the result as output. *******************************************************************************/ @Override public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException @@ -68,16 +68,16 @@ public class BasicETLCollectSourceFileNamesFunction implements BackendStep /******************************************************************************* - ** define the metaData that describes this function + ** define the metaData that describes this step *******************************************************************************/ public QBackendStepMetaData defineStepMetaData() { return (new QBackendStepMetaData() - .withName(FUNCTION_NAME) + .withName(STEP_NAME) .withCode(new QCodeReference() .withName(this.getClass().getName()) .withCodeType(QCodeType.JAVA) - .withCodeUsage(QCodeUsage.FUNCTION)) + .withCodeUsage(QCodeUsage.BACKEND_STEP)) .withOutputMetaData(new QFunctionOutputMetaData() .addField(new QFieldMetaData(FIELD_SOURCE_FILE_PATHS, QFieldType.STRING)))); } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java index 39a708fb..aca2347a 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java @@ -64,12 +64,12 @@ public class FilesystemSyncProcess *******************************************************************************/ public QProcessMetaData defineProcessMetaData() { - QBackendStepMetaData syncFunction = new QBackendStepMetaData() - .withName(FilesystemSyncFunction.FUNCTION_NAME) + QBackendStepMetaData syncStep = new QBackendStepMetaData() + .withName(FilesystemSyncStep.STEP_NAME) .withCode(new QCodeReference() - .withName(FilesystemSyncFunction.class.getName()) + .withName(FilesystemSyncStep.class.getName()) .withCodeType(QCodeType.JAVA) - .withCodeUsage(QCodeUsage.FUNCTION)) + .withCodeUsage(QCodeUsage.BACKEND_STEP)) .withInputData(new QFunctionInputMetaData() .addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)) .addField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING)) @@ -78,6 +78,6 @@ public class FilesystemSyncProcess return new QProcessMetaData() .withName(PROCESS_NAME) - .addStep(syncFunction); + .addStep(syncStep); } } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java similarity index 93% rename from src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java rename to src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java index 145edf6a..bedf2616 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java @@ -42,21 +42,21 @@ import org.apache.logging.log4j.Logger; /******************************************************************************* - ** Function body for collecting the file names that were discovered in the - ** Extract step. These will be lost during the transform, so we capture them here, - ** so that our Clean function can move or delete them. + ** BackendStep to sync two filesystem tables (copying the new files to a 3rd + ** location as well...) ** *******************************************************************************/ -public class FilesystemSyncFunction implements BackendStep +@SuppressWarnings("unchecked") +public class FilesystemSyncStep implements BackendStep { - private static final Logger LOG = LogManager.getLogger(FilesystemSyncFunction.class); + private static final Logger LOG = LogManager.getLogger(FilesystemSyncStep.class); - public static final String FUNCTION_NAME = "sync"; + public static final String STEP_NAME = "sync"; /******************************************************************************* - ** Execute the function - using the request as input, and the result as output. + ** Execute the step - using the request as input, and the result as output. *******************************************************************************/ @Override public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java similarity index 93% rename from src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java rename to src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java index 5a4e9fd2..e3a508d4 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java @@ -48,7 +48,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* ** Unit test for BasicETLCleanupSourceFilesFunction *******************************************************************************/ -public class BasicETLCleanupSourceFilesFunctionTest +public class BasicETLCleanupSourceFilesStepTest { /******************************************************************************* @@ -137,12 +137,12 @@ public class BasicETLCleanupSourceFilesFunctionTest private void testDelete(QInstance qInstance, List filePaths) throws Exception { RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of( - BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_DELETE, + BasicETLCleanupSourceFilesStep.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesStep.VALUE_DELETE, // todo - even though this field isn't needed, since we gave a value of "delete" // the RunFunctionAction considers any missing input to be an error... - BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, "")); + BasicETLCleanupSourceFilesStep.FIELD_DESTINATION_FOR_MOVES, "")); - assertNull(runBackendStepResult.getError()); + assertNull(runBackendStepResult.getException()); for(String filePath : filePaths) { assertFalse(new File(filePath).exists(), "File should have been deleted."); @@ -158,10 +158,10 @@ public class BasicETLCleanupSourceFilesFunctionTest { String trashDir = File.separator + "tmp" + File.separator + "trash"; RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of( - BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_MOVE, - BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir)); + BasicETLCleanupSourceFilesStep.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesStep.VALUE_MOVE, + BasicETLCleanupSourceFilesStep.FIELD_DESTINATION_FOR_MOVES, trashDir)); - assertNull(runBackendStepResult.getError()); + assertNull(runBackendStepResult.getException()); for(String filePath : filePaths) { @@ -179,7 +179,7 @@ public class BasicETLCleanupSourceFilesFunctionTest *******************************************************************************/ private RunBackendStepResult runFunction(QInstance qInstance, List filePaths, Map values) throws Exception { - QBackendStepMetaData backendStepMetaData = new BasicETLCleanupSourceFilesFunction().defineStepMetaData(); + QBackendStepMetaData backendStepMetaData = new BasicETLCleanupSourceFilesStep().defineStepMetaData(); QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData); qInstance.addProcess(qProcessMetaData); @@ -200,7 +200,7 @@ public class BasicETLCleanupSourceFilesFunctionTest runBackendStepRequest.setSession(TestUtils.getMockSession()); runBackendStepRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS); runBackendStepRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3); - runBackendStepRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet)); + runBackendStepRequest.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet)); for(Map.Entry entry : values.entrySet()) { diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunctionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStepTest.java similarity index 97% rename from src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunctionTest.java rename to src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStepTest.java index eeff6da6..db56b418 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunctionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStepTest.java @@ -41,7 +41,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* ** Unit test for BasicETLCollectSourceFileNamesFunction *******************************************************************************/ -class BasicETLCollectSourceFileNamesFunctionTest +class BasicETLCollectSourceFileNamesStepTest { /******************************************************************************* @@ -94,7 +94,7 @@ class BasicETLCollectSourceFileNamesFunctionTest private String runTest(String... fileNames) throws Exception { QInstance qInstance = TestUtils.defineInstance(); - QBackendStepMetaData backendStepMetaData = new BasicETLCollectSourceFileNamesFunction().defineStepMetaData(); + QBackendStepMetaData backendStepMetaData = new BasicETLCollectSourceFileNamesStep().defineStepMetaData(); QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData); qInstance.addProcess(qProcessMetaData); @@ -110,6 +110,6 @@ class BasicETLCollectSourceFileNamesFunctionTest RunBackendStepAction runBackendStepAction = new RunBackendStepAction(); RunBackendStepResult result = runBackendStepAction.execute(runBackendStepRequest); - return ((String) result.getValues().get(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS)); + return ((String) result.getValues().get(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS)); } } \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java index 5fc19bda..4b3d87b7 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java @@ -81,7 +81,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv"); QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); - QBackendStepMetaData step = process.getBackendStep(FilesystemSyncFunction.FUNCTION_NAME); + QBackendStepMetaData step = process.getBackendStep(FilesystemSyncStep.STEP_NAME); qInstance.addProcess(process); step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); @@ -109,7 +109,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test RunBackendStepAction runFunctionAction = new RunBackendStepAction(); RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest); - System.out.println(runBackendStepResult); + // System.out.println(runBackendStepResult); printTableListing(sourceBackend, sourceTable); printTableListing(archiveBackend, archiveTable); @@ -144,7 +144,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv"); QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); - QBackendStepMetaData step = process.getBackendStep(FilesystemSyncFunction.FUNCTION_NAME); + QBackendStepMetaData step = process.getBackendStep(FilesystemSyncStep.STEP_NAME); qInstance.addProcess(process); step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); @@ -172,7 +172,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test RunBackendStepAction runFunctionAction = new RunBackendStepAction(); RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest); - System.out.println(runBackendStepResult); + // System.out.println(runBackendStepResult); printTableListing(vendorBackend, sourceTable); printTableListing(localBackend, archiveTable); diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java index 2bcb5158..9f0e3933 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java @@ -58,7 +58,7 @@ class FilesystemSyncProcessTest QTableMetaData archiveTable = defineTable("archive"); QTableMetaData processingTable = defineTable("processing"); QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); - QBackendStepMetaData step = (QBackendStepMetaData) process.getStep(FilesystemSyncFunction.FUNCTION_NAME); + QBackendStepMetaData step = (QBackendStepMetaData) process.getStep(FilesystemSyncStep.STEP_NAME); step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); @@ -90,7 +90,7 @@ class FilesystemSyncProcessTest RunBackendStepAction runFunctionAction = new RunBackendStepAction(); RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest); - System.out.println(runBackendStepResult); + // System.out.println(runBackendStepResult); }