QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/QQQ-21-processes-in-ui' of github.com:Kingsrook/qqq-backend-module-filesystem into feature/QQQ-21-processes-in-ui

Browse Source
This commit is contained in:
Tim Chamberlain
2022-07-08 15:18:47 -05:00
parent 7330a59417 68777bd253
commit 36003f1242
9 changed files with 51 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
checkstyle.xml
View File

@ -181,8 +181,8 @@
</module>
-->
<module name="OverloadMethodsDeclarationOrder"/>
<module name="VariableDeclarationUsageDistance"/>
<!--
<module name="VariableDeclarationUsageDistance"/>
<module name="CustomImportOrder">
<property name="sortImportsInGroupAlphabetically" value="true"/>
<property name="separateLineBetweenGroups" value="true"/>

22
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java → src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java
View File

@ -47,24 +47,24 @@ import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Function body for performing the Cleanup step of a basic ETL process - e.g.,
** BackendStep for performing the Cleanup step of a basic ETL process - e.g.,
** after the loading, delete or move the processed file(s).
*******************************************************************************/
public class BasicETLCleanupSourceFilesFunction implements BackendStep
public class BasicETLCleanupSourceFilesStep implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesFunction.class);
private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesStep.class);
public static final String FIELD_MOVE_OR_DELETE = "moveOrDelete";
public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves";
public static final String VALUE_MOVE = "move";
public static final String VALUE_DELETE = "delete";
public static final String FUNCTION_NAME = "cleanupSourceFiles";
public static final String STEP_NAME = "cleanupSourceFiles";
/*******************************************************************************
** Execute the function - using the request as input, and the result as output.
** Execute the step - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
@ -76,15 +76,15 @@ public class BasicETLCleanupSourceFilesFunction implements BackendStep
if(!(module instanceof FilesystemBackendModuleInterface filesystemModule))
{
throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function."));
throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this action."));
}
AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase();
actionBase.preAction(backend);
String sourceFilePaths = runBackendStepRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS);
String sourceFilePaths = runBackendStepRequest.getValueString(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS);
if(!StringUtils.hasContent(sourceFilePaths))
{
LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS + "]");
LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS + "]");
return;
}
@ -118,16 +118,16 @@ public class BasicETLCleanupSourceFilesFunction implements BackendStep
/*******************************************************************************
** define the metaData that describes this function
** define the metaData that describes this step
*******************************************************************************/
public QBackendStepMetaData defineStepMetaData()
{
return (new QBackendStepMetaData()
.withName(FUNCTION_NAME)
.withName(STEP_NAME)
.withCode(new QCodeReference()
.withName(this.getClass().getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.FUNCTION))
.withCodeUsage(QCodeUsage.BACKEND_STEP))
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData("moveOrDelete", QFieldType.STRING))
.addField(new QFieldMetaData("destinationForMoves", QFieldType.STRING))));

16
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java → src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java
View File

@ -40,21 +40,21 @@ import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendD
/*******************************************************************************
** Function body for collecting the file names that were discovered in the
** BackendStep for collecting the file names that were discovered in the
** Extract step. These will be lost during the transform, so we capture them here,
** so that our Clean function can move or delete them.
** so that our Clean step can move or delete them.
**
** TODO - need unit test!!
*******************************************************************************/
public class BasicETLCollectSourceFileNamesFunction implements BackendStep
public class BasicETLCollectSourceFileNamesStep implements BackendStep
{
public static final String FUNCTION_NAME = "collectSourceFileNames";
public static final String STEP_NAME = "collectSourceFileNames";
public static final String FIELD_SOURCE_FILE_PATHS = "sourceFilePaths";
/*******************************************************************************
** Execute the function - using the request as input, and the result as output.
** Execute the step - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
@ -68,16 +68,16 @@ public class BasicETLCollectSourceFileNamesFunction implements BackendStep
/*******************************************************************************
** define the metaData that describes this function
** define the metaData that describes this step
*******************************************************************************/
public QBackendStepMetaData defineStepMetaData()
{
return (new QBackendStepMetaData()
.withName(FUNCTION_NAME)
.withName(STEP_NAME)
.withCode(new QCodeReference()
.withName(this.getClass().getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.FUNCTION))
.withCodeUsage(QCodeUsage.BACKEND_STEP))
.withOutputMetaData(new QFunctionOutputMetaData()
.addField(new QFieldMetaData(FIELD_SOURCE_FILE_PATHS, QFieldType.STRING))));
}

10
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java
View File

@ -64,12 +64,12 @@ public class FilesystemSyncProcess
*******************************************************************************/
public QProcessMetaData defineProcessMetaData()
{
QBackendStepMetaData syncFunction = new QBackendStepMetaData()
.withName(FilesystemSyncFunction.FUNCTION_NAME)
QBackendStepMetaData syncStep = new QBackendStepMetaData()
.withName(FilesystemSyncStep.STEP_NAME)
.withCode(new QCodeReference()
.withName(FilesystemSyncFunction.class.getName())
.withName(FilesystemSyncStep.class.getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.FUNCTION))
.withCodeUsage(QCodeUsage.BACKEND_STEP))
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING))
.addField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING))
@ -78,6 +78,6 @@ public class FilesystemSyncProcess
return new QProcessMetaData()
.withName(PROCESS_NAME)
.addStep(syncFunction);
.addStep(syncStep);
}
}

14
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java → src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java
View File

@ -42,21 +42,21 @@ import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Function body for collecting the file names that were discovered in the
** Extract step. These will be lost during the transform, so we capture them here,
** so that our Clean function can move or delete them.
** BackendStep to sync two filesystem tables (copying the new files to a 3rd
** location as well...)
**
*******************************************************************************/
public class FilesystemSyncFunction implements BackendStep
@SuppressWarnings("unchecked")
public class FilesystemSyncStep implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(FilesystemSyncFunction.class);
private static final Logger LOG = LogManager.getLogger(FilesystemSyncStep.class);
public static final String FUNCTION_NAME = "sync";
public static final String STEP_NAME = "sync";
/*******************************************************************************
** Execute the function - using the request as input, and the result as output.
** Execute the step - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException

18
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java → src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java
View File

@ -48,7 +48,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for BasicETLCleanupSourceFilesFunction
*******************************************************************************/
public class BasicETLCleanupSourceFilesFunctionTest
public class BasicETLCleanupSourceFilesStepTest
{
/*******************************************************************************
@ -137,12 +137,12 @@ public class BasicETLCleanupSourceFilesFunctionTest
private void testDelete(QInstance qInstance, List<String> filePaths) throws Exception
{
RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_DELETE,
BasicETLCleanupSourceFilesStep.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesStep.VALUE_DELETE,
// todo - even though this field isn't needed, since we gave a value of "delete"
// the RunFunctionAction considers any missing input to be an error...
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, ""));
BasicETLCleanupSourceFilesStep.FIELD_DESTINATION_FOR_MOVES, ""));
assertNull(runBackendStepResult.getError());
assertNull(runBackendStepResult.getException());
for(String filePath : filePaths)
{
assertFalse(new File(filePath).exists(), "File should have been deleted.");
@ -158,10 +158,10 @@ public class BasicETLCleanupSourceFilesFunctionTest
{
String trashDir = File.separator + "tmp" + File.separator + "trash";
RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_MOVE,
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir));
BasicETLCleanupSourceFilesStep.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesStep.VALUE_MOVE,
BasicETLCleanupSourceFilesStep.FIELD_DESTINATION_FOR_MOVES, trashDir));
assertNull(runBackendStepResult.getError());
assertNull(runBackendStepResult.getException());
for(String filePath : filePaths)
{
@ -179,7 +179,7 @@ public class BasicETLCleanupSourceFilesFunctionTest
*******************************************************************************/
private RunBackendStepResult runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception
{
QBackendStepMetaData backendStepMetaData = new BasicETLCleanupSourceFilesFunction().defineStepMetaData();
QBackendStepMetaData backendStepMetaData = new BasicETLCleanupSourceFilesStep().defineStepMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
qInstance.addProcess(qProcessMetaData);
@ -200,7 +200,7 @@ public class BasicETLCleanupSourceFilesFunctionTest
runBackendStepRequest.setSession(TestUtils.getMockSession());
runBackendStepRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
runBackendStepRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
runBackendStepRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
runBackendStepRequest.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
for(Map.Entry<String, String> entry : values.entrySet())
{

6
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunctionTest.java → src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStepTest.java
View File

@ -41,7 +41,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for BasicETLCollectSourceFileNamesFunction
*******************************************************************************/
class BasicETLCollectSourceFileNamesFunctionTest
class BasicETLCollectSourceFileNamesStepTest
{
/*******************************************************************************
@ -94,7 +94,7 @@ class BasicETLCollectSourceFileNamesFunctionTest
private String runTest(String... fileNames) throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QBackendStepMetaData backendStepMetaData = new BasicETLCollectSourceFileNamesFunction().defineStepMetaData();
QBackendStepMetaData backendStepMetaData = new BasicETLCollectSourceFileNamesStep().defineStepMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
qInstance.addProcess(qProcessMetaData);
@ -110,6 +110,6 @@ class BasicETLCollectSourceFileNamesFunctionTest
RunBackendStepAction runBackendStepAction = new RunBackendStepAction();
RunBackendStepResult result = runBackendStepAction.execute(runBackendStepRequest);
return ((String) result.getValues().get(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS));
return ((String) result.getValues().get(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS));
}
}

8
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java
View File

@ -81,7 +81,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QBackendStepMetaData step = process.getBackendStep(FilesystemSyncFunction.FUNCTION_NAME);
QBackendStepMetaData step = process.getBackendStep(FilesystemSyncStep.STEP_NAME);
qInstance.addProcess(process);
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
@ -109,7 +109,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
System.out.println(runBackendStepResult);
// System.out.println(runBackendStepResult);
printTableListing(sourceBackend, sourceTable);
printTableListing(archiveBackend, archiveTable);
@ -144,7 +144,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QBackendStepMetaData step = process.getBackendStep(FilesystemSyncFunction.FUNCTION_NAME);
QBackendStepMetaData step = process.getBackendStep(FilesystemSyncStep.STEP_NAME);
qInstance.addProcess(process);
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
@ -172,7 +172,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
System.out.println(runBackendStepResult);
// System.out.println(runBackendStepResult);
printTableListing(vendorBackend, sourceTable);
printTableListing(localBackend, archiveTable);

4
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java
View File

@ -58,7 +58,7 @@ class FilesystemSyncProcessTest
QTableMetaData archiveTable = defineTable("archive");
QTableMetaData processingTable = defineTable("processing");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QBackendStepMetaData step = (QBackendStepMetaData) process.getStep(FilesystemSyncFunction.FUNCTION_NAME);
QBackendStepMetaData step = (QBackendStepMetaData) process.getStep(FilesystemSyncStep.STEP_NAME);
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
@ -90,7 +90,7 @@ class FilesystemSyncProcessTest
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
System.out.println(runBackendStepResult);
// System.out.println(runBackendStepResult);
}