QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

QQQ-21 updated per updated backend-core - process function -> step renames

Browse Source
This commit is contained in:
darin.kelkhoff
2022-07-06 13:53:33 -05:00
parent 2deaf9cc8e
commit 1252bf5af2
9 changed files with 119 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -51,7 +51,7 @@
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-backend-core</artifactId>
<version>0.0.0</version>
<version>0.1.0-20220706.184937-2</version>
</dependency>
<!-- 3rd party deps specifically for this module -->

32
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java
View File

@ -24,9 +24,9 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et
import java.io.File;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
@ -34,8 +34,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
@ -50,7 +50,7 @@ import org.apache.logging.log4j.Logger;
** Function body for performing the Cleanup step of a basic ETL process - e.g.,
** after the loading, delete or move the processed file(s).
*******************************************************************************/
public class BasicETLCleanupSourceFilesFunction implements FunctionBody
public class BasicETLCleanupSourceFilesFunction implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesFunction.class);
@ -67,11 +67,11 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
** Execute the function - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
{
String sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
QTableMetaData table = runFunctionRequest.getInstance().getTable(sourceTableName);
QBackendMetaData backend = runFunctionRequest.getInstance().getBackendForTable(sourceTableName);
String sourceTableName = runBackendStepRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
QTableMetaData table = runBackendStepRequest.getInstance().getTable(sourceTableName);
QBackendMetaData backend = runBackendStepRequest.getInstance().getBackendForTable(sourceTableName);
QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend);
if(!(module instanceof FilesystemBackendModuleInterface filesystemModule))
@ -81,7 +81,7 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase();
actionBase.preAction(backend);
String sourceFilePaths = runFunctionRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS);
String sourceFilePaths = runBackendStepRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS);
if(!StringUtils.hasContent(sourceFilePaths))
{
LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS + "]");
@ -91,21 +91,21 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
String[] sourceFiles = sourceFilePaths.split(",");
for(String sourceFile : sourceFiles)
{
String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE);
String moveOrDelete = runBackendStepRequest.getValueString(FIELD_MOVE_OR_DELETE);
if(VALUE_DELETE.equals(moveOrDelete))
{
actionBase.deleteFile(runFunctionRequest.getInstance(), table, sourceFile);
actionBase.deleteFile(runBackendStepRequest.getInstance(), table, sourceFile);
}
else if(VALUE_MOVE.equals(moveOrDelete))
{
String destinationForMoves = runFunctionRequest.getValueString(FIELD_DESTINATION_FOR_MOVES);
String destinationForMoves = runBackendStepRequest.getValueString(FIELD_DESTINATION_FOR_MOVES);
if(!StringUtils.hasContent(destinationForMoves))
{
throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value."));
}
String filePathWithoutBase = actionBase.stripBackendAndTableBasePathsFromFileName(sourceFile, backend, table);
String destinationPath = destinationForMoves + File.separator + filePathWithoutBase;
actionBase.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath);
actionBase.moveFile(runBackendStepRequest.getInstance(), table, sourceFile, destinationPath);
}
else
{
@ -120,9 +120,9 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
/*******************************************************************************
** define the metaData that describes this function
*******************************************************************************/
public QFunctionMetaData defineFunctionMetaData()
public QBackendStepMetaData defineStepMetaData()
{
return (new QFunctionMetaData()
return (new QBackendStepMetaData()
.withName(FUNCTION_NAME)
.withCode(new QCodeReference()
.withName(this.getClass().getName())

20
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java
View File

@ -25,15 +25,15 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
@ -46,7 +46,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendD
**
** TODO - need unit test!!
*******************************************************************************/
public class BasicETLCollectSourceFileNamesFunction implements FunctionBody
public class BasicETLCollectSourceFileNamesFunction implements BackendStep
{
public static final String FUNCTION_NAME = "collectSourceFileNames";
public static final String FIELD_SOURCE_FILE_PATHS = "sourceFilePaths";
@ -57,12 +57,12 @@ public class BasicETLCollectSourceFileNamesFunction implements FunctionBody
** Execute the function - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
{
Set<String> sourceFiles = runFunctionRequest.getRecords().stream()
Set<String> sourceFiles = runBackendStepRequest.getRecords().stream()
.map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))
.collect(Collectors.toSet());
runFunctionResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles));
runBackendStepResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles));
}
@ -70,9 +70,9 @@ public class BasicETLCollectSourceFileNamesFunction implements FunctionBody
/*******************************************************************************
** define the metaData that describes this function
*******************************************************************************/
public QFunctionMetaData defineFunctionMetaData()
public QBackendStepMetaData defineStepMetaData()
{
return (new QFunctionMetaData()
return (new QBackendStepMetaData()
.withName(FUNCTION_NAME)
.withCode(new QCodeReference()
.withName(this.getClass().getName())

24
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java
View File

@ -29,9 +29,9 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
@ -47,7 +47,7 @@ import org.apache.logging.log4j.Logger;
** so that our Clean function can move or delete them.
**
*******************************************************************************/
public class FilesystemSyncFunction implements FunctionBody
public class FilesystemSyncFunction implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(FilesystemSyncFunction.class);
@ -59,30 +59,30 @@ public class FilesystemSyncFunction implements FunctionBody
** Execute the function - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
{
QTableMetaData sourceTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
QTableMetaData archiveTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
QTableMetaData sourceTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
QTableMetaData archiveTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
QTableMetaData processingTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName());
QBackendMetaData sourceBackend = runBackendStepRequest.getInstance().getBackendForTable(sourceTable.getName());
FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase();
sourceActionBase.preAction(sourceBackend);
Map<String, Object> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName());
QBackendMetaData archiveBackend = runBackendStepRequest.getInstance().getBackendForTable(archiveTable.getName());
FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase();
archiveActionBase.preAction(archiveBackend);
Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet();
QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName());
QBackendMetaData processingBackend = runBackendStepRequest.getInstance().getBackendForTable(processingTable.getName());
FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase();
processingActionBase.preAction(processingBackend);
Integer maxFilesToSync = runFunctionRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
Integer maxFilesToSync = runBackendStepRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
int syncedFileCount = 0;
for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet())
{

6
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java
View File

@ -27,8 +27,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
@ -64,7 +64,7 @@ public class FilesystemSyncProcess
*******************************************************************************/
public QProcessMetaData defineProcessMetaData()
{
QFunctionMetaData syncFunction = new QFunctionMetaData()
QBackendStepMetaData syncFunction = new QBackendStepMetaData()
.withName(FilesystemSyncFunction.FUNCTION_NAME)
.withCode(new QCodeReference()
.withName(FilesystemSyncFunction.class.getName())
@ -78,6 +78,6 @@ public class FilesystemSyncProcess
return new QProcessMetaData()
.withName(PROCESS_NAME)
.addFunction(syncFunction);
.addStep(syncFunction);
}
}

42
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java
View File

@ -27,11 +27,11 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -136,13 +136,13 @@ public class BasicETLCleanupSourceFilesFunctionTest
*******************************************************************************/
private void testDelete(QInstance qInstance, List<String> filePaths) throws Exception
{
RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of(
RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_DELETE,
// todo - even though this field isn't needed, since we gave a value of "delete"
// the RunFunctionAction considers any missing input to be an error...
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, ""));
assertNull(runFunctionResult.getError());
assertNull(runBackendStepResult.getError());
for(String filePath : filePaths)
{
assertFalse(new File(filePath).exists(), "File should have been deleted.");
@ -157,11 +157,11 @@ public class BasicETLCleanupSourceFilesFunctionTest
private void testMove(QInstance qInstance, List<String> filePaths) throws Exception
{
String trashDir = File.separator + "tmp" + File.separator + "trash";
RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of(
RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_MOVE,
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir));
assertNull(runFunctionResult.getError());
assertNull(runBackendStepResult.getError());
for(String filePath : filePaths)
{
@ -177,10 +177,10 @@ public class BasicETLCleanupSourceFilesFunctionTest
/*******************************************************************************
**
*******************************************************************************/
private RunFunctionResult runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception
private RunBackendStepResult runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception
{
QFunctionMetaData qFunctionMetaData = new BasicETLCleanupSourceFilesFunction().defineFunctionMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addFunction(qFunctionMetaData);
QBackendStepMetaData backendStepMetaData = new BasicETLCleanupSourceFilesFunction().defineStepMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
qInstance.addProcess(qProcessMetaData);
HashSet<String> filePathsSet = new HashSet<>(filePaths);
@ -193,22 +193,22 @@ public class BasicETLCleanupSourceFilesFunctionTest
// List<QRecord> records = filePaths.stream()
// .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList();
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(qFunctionMetaData.getName());
runFunctionRequest.setProcessName(qProcessMetaData.getName());
RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runBackendStepRequest.setStepName(backendStepMetaData.getName());
runBackendStepRequest.setProcessName(qProcessMetaData.getName());
// runFunctionRequest.setRecords(records);
runFunctionRequest.setSession(TestUtils.getMockSession());
runFunctionRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
runFunctionRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
runFunctionRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
runBackendStepRequest.setSession(TestUtils.getMockSession());
runBackendStepRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
runBackendStepRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
runBackendStepRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
for(Map.Entry<String, String> entry : values.entrySet())
{
runFunctionRequest.addValue(entry.getKey(), entry.getValue());
runBackendStepRequest.addValue(entry.getKey(), entry.getValue());
}
RunFunctionAction runFunctionAction = new RunFunctionAction();
return (runFunctionAction.execute(runFunctionRequest));
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
return (runFunctionAction.execute(runBackendStepRequest));
}

64
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java
View File

@ -25,16 +25,16 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi
import java.util.List;
import java.util.stream.Collectors;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -81,12 +81,12 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
QBackendStepMetaData step = process.getBackendStep(FilesystemSyncFunction.FUNCTION_NAME);
qInstance.addProcess(process);
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
///////////////////////////
// write some test files //
@ -99,17 +99,17 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
printTableListing(archiveBackend, archiveTable);
printTableListing(processingBackend, processingTable);
//////////////////////
// run the function //
//////////////////////
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(function.getName());
runFunctionRequest.setProcessName(process.getName());
runFunctionRequest.setSession(TestUtils.getMockSession());
//////////////////
// run the step //
//////////////////
RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runBackendStepRequest.setStepName(step.getName());
runBackendStepRequest.setProcessName(process.getName());
runBackendStepRequest.setSession(TestUtils.getMockSession());
RunFunctionAction runFunctionAction = new RunFunctionAction();
RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
System.out.println(runFunctionResult);
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
System.out.println(runBackendStepResult);
printTableListing(sourceBackend, sourceTable);
printTableListing(archiveBackend, archiveTable);
@ -144,12 +144,12 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
QBackendStepMetaData step = process.getBackendStep(FilesystemSyncFunction.FUNCTION_NAME);
qInstance.addProcess(process);
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
///////////////////////////
// write some test files //
@ -162,17 +162,17 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
printTableListing(localBackend, archiveTable);
printTableListing(localBackend, processingTable);
//////////////////////
// run the function //
//////////////////////
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(function.getName());
runFunctionRequest.setProcessName(process.getName());
runFunctionRequest.setSession(TestUtils.getMockSession());
//////////////////
// run the step //
//////////////////
RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runBackendStepRequest.setStepName(step.getName());
runBackendStepRequest.setProcessName(process.getName());
runBackendStepRequest.setSession(TestUtils.getMockSession());
RunFunctionAction runFunctionAction = new RunFunctionAction();
RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
System.out.println(runFunctionResult);
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
System.out.println(runBackendStepResult);
printTableListing(vendorBackend, sourceTable);
printTableListing(localBackend, archiveTable);

34
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java
View File

@ -24,14 +24,14 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi
import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
@ -58,12 +58,12 @@ class FilesystemSyncProcessTest
QTableMetaData archiveTable = defineTable("archive");
QTableMetaData processingTable = defineTable("processing");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
QBackendStepMetaData step = (QBackendStepMetaData) process.getStep(FilesystemSyncFunction.FUNCTION_NAME);
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
// function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1);
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
// step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1);
QInstance qInstance = TestUtils.defineInstance();
qInstance.addTable(sourceTable);
@ -81,16 +81,16 @@ class FilesystemSyncProcessTest
writeTestFile(basePath, archiveTable, "2.txt", "x");
//////////////////////
// run the function //
// run the step //
//////////////////////
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(function.getName());
runFunctionRequest.setProcessName(process.getName());
runFunctionRequest.setSession(TestUtils.getMockSession());
RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runBackendStepRequest.setStepName(step.getName());
runBackendStepRequest.setProcessName(process.getName());
runBackendStepRequest.setSession(TestUtils.getMockSession());
RunFunctionAction runFunctionAction = new RunFunctionAction();
RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
System.out.println(runFunctionResult);
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
System.out.println(runBackendStepResult);
}

2
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java
View File

@ -52,7 +52,7 @@ class S3BackendMetaDataTest
System.out.println(JsonUtils.prettyPrint(json));
System.out.println(json);
String expectToContain = """
{"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","secretKey":null,"accessKey":null,"backendType":"s3","name":"s3","region":null}""";
{"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","backendType":"s3","name":"s3"}""";
assertTrue(json.contains(expectToContain));
}