QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2 from Kingsrook/feature/QQQ-21-processes-in-ui

Browse Source 
Feature/qqq 21 processes in UI
This commit is contained in:
darin.kelkhoff
2022-07-14 09:45:41 -05:00
committed by GitHub
parent 2deaf9cc8e 36003f1242
commit b83165656d
17 changed files with 472 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.circleci/config.yml
View File

@ -57,7 +57,7 @@ jobs:
- localstack/startup
- install_java17
- run_maven:
maven_subcommand: test
maven_subcommand: verify
- slack/notify:
event: fail

2
checkstyle.xml
View File

@ -181,8 +181,8 @@
</module>
-->
<module name="OverloadMethodsDeclarationOrder"/>
<module name="VariableDeclarationUsageDistance"/>
<!--
<module name="VariableDeclarationUsageDistance"/>
<module name="CustomImportOrder">
<property name="sortImportsInGroupAlphabetically" value="true"/>
<property name="separateLineBetweenGroups" value="true"/>

86
pom.xml
View File

@ -44,6 +44,8 @@
<maven.compiler.target>17</maven.compiler.target>
<maven.compiler.showDeprecation>true</maven.compiler.showDeprecation>
<maven.compiler.showWarnings>true</maven.compiler.showWarnings>
<coverage.haltOnFailure>true</coverage.haltOnFailure>
<coverage.instructionCoveredRatioMinimum>0.80</coverage.instructionCoveredRatioMinimum>
</properties>
<dependencies>
@ -51,7 +53,7 @@
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-backend-core</artifactId>
<version>0.0.0</version>
<version>0.1.0-20220708.195335-5</version>
</dependency>
<!-- 3rd party deps specifically for this module -->
@ -115,6 +117,9 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.0.0-M5</version>
<configuration>
<argLine>@{jaCoCoArgLine}</argLine>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@ -164,7 +169,84 @@
<versionDigitToIncrement>1</versionDigitToIncrement> <!-- In general, we update the minor -->
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>pre-unit-test</id>
<goals>
<goal>prepare-agent</goal>
</goals>
<configuration>
<propertyName>jaCoCoArgLine</propertyName>
</configuration>
</execution>
<execution>
<id>unit-test-check</id>
<goals>
<goal>check</goal>
</goals>
<configuration>
<!-- Gives us the ability to pass a parameter to not fail due to coverage E.g. -Dcoverage.haltOnFailure=false -->
<haltOnFailure>${coverage.haltOnFailure}</haltOnFailure>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>${coverage.instructionCoveredRatioMinimum}</minimum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
<execution>
<id>post-unit-test</id>
<phase>verify</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>exec-maven-plugin</artifactId>
<groupId>org.codehaus.mojo</groupId>
<version>3.0.0</version>
<executions>
<execution>
<id>test-coverage-summary</id>
<phase>verify</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>sh</executable>
<arguments>
<argument>-c</argument>
<argument>
<![CDATA[
echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
xpath -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
echo
echo "Jacoco coverage summary report:"
echo " See also target/site/jacoco/index.html"
echo " and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
echo "------------------------------------------------------------"
paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
rm /tmp/$$.headers /tmp/$$.values
]]>
</argument>
</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

2
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java
View File

@ -31,6 +31,8 @@ import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFile
*******************************************************************************/
public interface FilesystemBackendModuleInterface<FILE>
{
String CUSTOMIZER_FILE_POST_FILE_READ = "postFileRead";
/*******************************************************************************
** For filesystem backends, get the module-specific action base-class, that helps
** with functions like listing and deleting files.

3
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
View File

@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
@ -269,7 +270,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
*******************************************************************************/
private String customizeFileContentsAfterReading(QTableMetaData table, String fileContents) throws QException
{
Optional<QCodeReference> optionalCustomizer = table.getCustomizer("postFileRead");
Optional<QCodeReference> optionalCustomizer = table.getCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ);
if(optionalCustomizer.isEmpty())
{
return (fileContents);

52
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java → src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java
View File

@ -24,9 +24,9 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et
import java.io.File;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
@ -34,8 +34,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
@ -47,65 +47,65 @@ import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Function body for performing the Cleanup step of a basic ETL process - e.g.,
** BackendStep for performing the Cleanup step of a basic ETL process - e.g.,
** after the loading, delete or move the processed file(s).
*******************************************************************************/
public class BasicETLCleanupSourceFilesFunction implements FunctionBody
public class BasicETLCleanupSourceFilesStep implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesFunction.class);
private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesStep.class);
public static final String FIELD_MOVE_OR_DELETE = "moveOrDelete";
public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves";
public static final String VALUE_MOVE = "move";
public static final String VALUE_DELETE = "delete";
public static final String FUNCTION_NAME = "cleanupSourceFiles";
public static final String VALUE_DELETE = "delete";
public static final String STEP_NAME = "cleanupSourceFiles";
/*******************************************************************************
** Execute the function - using the request as input, and the result as output.
** Execute the step - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
{
String sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
QTableMetaData table = runFunctionRequest.getInstance().getTable(sourceTableName);
QBackendMetaData backend = runFunctionRequest.getInstance().getBackendForTable(sourceTableName);
String sourceTableName = runBackendStepRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
QTableMetaData table = runBackendStepRequest.getInstance().getTable(sourceTableName);
QBackendMetaData backend = runBackendStepRequest.getInstance().getBackendForTable(sourceTableName);
QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend);
if(!(module instanceof FilesystemBackendModuleInterface filesystemModule))
{
throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function."));
throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this action."));
}
AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase();
actionBase.preAction(backend);
String sourceFilePaths = runFunctionRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS);
String sourceFilePaths = runBackendStepRequest.getValueString(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS);
if(!StringUtils.hasContent(sourceFilePaths))
{
LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS + "]");
LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS + "]");
return;
}
String[] sourceFiles = sourceFilePaths.split(",");
for(String sourceFile : sourceFiles)
{
String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE);
String moveOrDelete = runBackendStepRequest.getValueString(FIELD_MOVE_OR_DELETE);
if(VALUE_DELETE.equals(moveOrDelete))
{
actionBase.deleteFile(runFunctionRequest.getInstance(), table, sourceFile);
actionBase.deleteFile(runBackendStepRequest.getInstance(), table, sourceFile);
}
else if(VALUE_MOVE.equals(moveOrDelete))
{
String destinationForMoves = runFunctionRequest.getValueString(FIELD_DESTINATION_FOR_MOVES);
String destinationForMoves = runBackendStepRequest.getValueString(FIELD_DESTINATION_FOR_MOVES);
if(!StringUtils.hasContent(destinationForMoves))
{
throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value."));
}
String filePathWithoutBase = actionBase.stripBackendAndTableBasePathsFromFileName(sourceFile, backend, table);
String destinationPath = destinationForMoves + File.separator + filePathWithoutBase;
actionBase.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath);
actionBase.moveFile(runBackendStepRequest.getInstance(), table, sourceFile, destinationPath);
}
else
{
@ -118,16 +118,16 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
/*******************************************************************************
** define the metaData that describes this function
** define the metaData that describes this step
*******************************************************************************/
public QFunctionMetaData defineFunctionMetaData()
public QBackendStepMetaData defineStepMetaData()
{
return (new QFunctionMetaData()
.withName(FUNCTION_NAME)
return (new QBackendStepMetaData()
.withName(STEP_NAME)
.withCode(new QCodeReference()
.withName(this.getClass().getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.FUNCTION))
.withCodeUsage(QCodeUsage.BACKEND_STEP))
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData("moveOrDelete", QFieldType.STRING))
.addField(new QFieldMetaData("destinationForMoves", QFieldType.STRING))));

34
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java → src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java
View File

@ -25,59 +25,59 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
/*******************************************************************************
** Function body for collecting the file names that were discovered in the
** BackendStep for collecting the file names that were discovered in the
** Extract step. These will be lost during the transform, so we capture them here,
** so that our Clean function can move or delete them.
** so that our Clean step can move or delete them.
**
** TODO - need unit test!!
*******************************************************************************/
public class BasicETLCollectSourceFileNamesFunction implements FunctionBody
public class BasicETLCollectSourceFileNamesStep implements BackendStep
{
public static final String FUNCTION_NAME = "collectSourceFileNames";
public static final String STEP_NAME = "collectSourceFileNames";
public static final String FIELD_SOURCE_FILE_PATHS = "sourceFilePaths";
/*******************************************************************************
** Execute the function - using the request as input, and the result as output.
** Execute the step - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
{
Set<String> sourceFiles = runFunctionRequest.getRecords().stream()
Set<String> sourceFiles = runBackendStepRequest.getRecords().stream()
.map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))
.collect(Collectors.toSet());
runFunctionResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles));
runBackendStepResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles));
}
/*******************************************************************************
** define the metaData that describes this function
** define the metaData that describes this step
*******************************************************************************/
public QFunctionMetaData defineFunctionMetaData()
public QBackendStepMetaData defineStepMetaData()
{
return (new QFunctionMetaData()
.withName(FUNCTION_NAME)
return (new QBackendStepMetaData()
.withName(STEP_NAME)
.withCode(new QCodeReference()
.withName(this.getClass().getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.FUNCTION))
.withCodeUsage(QCodeUsage.BACKEND_STEP))
.withOutputMetaData(new QFunctionOutputMetaData()
.addField(new QFieldMetaData(FIELD_SOURCE_FILE_PATHS, QFieldType.STRING))));
}

12
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java
View File

@ -27,8 +27,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
@ -64,12 +64,12 @@ public class FilesystemSyncProcess
*******************************************************************************/
public QProcessMetaData defineProcessMetaData()
{
QFunctionMetaData syncFunction = new QFunctionMetaData()
.withName(FilesystemSyncFunction.FUNCTION_NAME)
QBackendStepMetaData syncStep = new QBackendStepMetaData()
.withName(FilesystemSyncStep.STEP_NAME)
.withCode(new QCodeReference()
.withName(FilesystemSyncFunction.class.getName())
.withName(FilesystemSyncStep.class.getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.FUNCTION))
.withCodeUsage(QCodeUsage.BACKEND_STEP))
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING))
.addField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING))
@ -78,6 +78,6 @@ public class FilesystemSyncProcess
return new QProcessMetaData()
.withName(PROCESS_NAME)
.addFunction(syncFunction);
.addStep(syncStep);
}
}

36
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java → src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java
View File

@ -29,9 +29,9 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
@ -42,47 +42,47 @@ import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Function body for collecting the file names that were discovered in the
** Extract step. These will be lost during the transform, so we capture them here,
** so that our Clean function can move or delete them.
** BackendStep to sync two filesystem tables (copying the new files to a 3rd
** location as well...)
**
*******************************************************************************/
public class FilesystemSyncFunction implements FunctionBody
@SuppressWarnings("unchecked")
public class FilesystemSyncStep implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(FilesystemSyncFunction.class);
private static final Logger LOG = LogManager.getLogger(FilesystemSyncStep.class);
public static final String FUNCTION_NAME = "sync";
public static final String STEP_NAME = "sync";
/*******************************************************************************
** Execute the function - using the request as input, and the result as output.
** Execute the step - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
{
QTableMetaData sourceTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
QTableMetaData archiveTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
QTableMetaData sourceTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
QTableMetaData archiveTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
QTableMetaData processingTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName());
QBackendMetaData sourceBackend = runBackendStepRequest.getInstance().getBackendForTable(sourceTable.getName());
FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase();
sourceActionBase.preAction(sourceBackend);
Map<String, Object> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName());
QBackendMetaData archiveBackend = runBackendStepRequest.getInstance().getBackendForTable(archiveTable.getName());
FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase();
archiveActionBase.preAction(archiveBackend);
Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet();
QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName());
QBackendMetaData processingBackend = runBackendStepRequest.getInstance().getBackendForTable(processingTable.getName());
FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase();
processingActionBase.preAction(processingBackend);
Integer maxFilesToSync = runFunctionRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
Integer maxFilesToSync = runBackendStepRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
int syncedFileCount = 0;
for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet())
{

15
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java
View File

@ -27,6 +27,7 @@ import java.io.InputStream;
import java.util.List;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@ -67,6 +68,17 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
return;
}
s3Utils = new S3Utils();
s3Utils.setAmazonS3(buildAmazonS3ClientFromBackendMetaData(backendMetaData));
}
/*******************************************************************************
**
*******************************************************************************/
protected AmazonS3 buildAmazonS3ClientFromBackendMetaData(QBackendMetaData backendMetaData)
{
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendMetaData);
AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard();
@ -81,8 +93,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
clientBuilder.setRegion(s3BackendMetaData.getRegion());
}
s3Utils = new S3Utils();
s3Utils.setAmazonS3(clientBuilder.build());
return (clientBuilder.build());
}

44
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java
View File

@ -22,11 +22,17 @@
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@ -44,8 +50,10 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
@Test
public void testQuery1() throws QException
{
QueryRequest queryRequest = initQueryRequest();
QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
QueryRequest queryRequest = new QueryRequest();
queryRequest.setInstance(TestUtils.defineInstance());
queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows");
Assertions.assertTrue(queryResult.getRecords().stream()
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
@ -57,12 +65,36 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
/*******************************************************************************
**
*******************************************************************************/
private QueryRequest initQueryRequest() throws QInstanceValidationException
@Test
public void testQueryWithFileCustomizer() throws QException
{
QueryRequest queryRequest = new QueryRequest();
queryRequest.setInstance(TestUtils.defineInstance());
QInstance instance = TestUtils.defineInstance();
QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
table.withCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ, new QCodeReference()
.withName(ValueUpshifter.class.getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.CUSTOMIZER));
queryRequest.setInstance(instance);
queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
return queryRequest;
QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows");
Assertions.assertTrue(
queryResult.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")),
"All records should have their email addresses up-shifted.");
}
public static class ValueUpshifter implements Function<String, String>
{
@Override
public String apply(String s)
{
return (s.replaceAll("kingsrook.com", "KINGSROOK.COM"));
}
}
}

52
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java → src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java
View File

@ -27,11 +27,11 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -48,7 +48,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for BasicETLCleanupSourceFilesFunction
*******************************************************************************/
public class BasicETLCleanupSourceFilesFunctionTest
public class BasicETLCleanupSourceFilesStepTest
{
/*******************************************************************************
@ -136,13 +136,13 @@ public class BasicETLCleanupSourceFilesFunctionTest
*******************************************************************************/
private void testDelete(QInstance qInstance, List<String> filePaths) throws Exception
{
RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_DELETE,
RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesStep.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesStep.VALUE_DELETE,
// todo - even though this field isn't needed, since we gave a value of "delete"
// the RunFunctionAction considers any missing input to be an error...
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, ""));
BasicETLCleanupSourceFilesStep.FIELD_DESTINATION_FOR_MOVES, ""));
assertNull(runFunctionResult.getError());
assertNull(runBackendStepResult.getException());
for(String filePath : filePaths)
{
assertFalse(new File(filePath).exists(), "File should have been deleted.");
@ -157,11 +157,11 @@ public class BasicETLCleanupSourceFilesFunctionTest
private void testMove(QInstance qInstance, List<String> filePaths) throws Exception
{
String trashDir = File.separator + "tmp" + File.separator + "trash";
RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_MOVE,
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir));
RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesStep.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesStep.VALUE_MOVE,
BasicETLCleanupSourceFilesStep.FIELD_DESTINATION_FOR_MOVES, trashDir));
assertNull(runFunctionResult.getError());
assertNull(runBackendStepResult.getException());
for(String filePath : filePaths)
{
@ -177,10 +177,10 @@ public class BasicETLCleanupSourceFilesFunctionTest
/*******************************************************************************
**
*******************************************************************************/
private RunFunctionResult runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception
private RunBackendStepResult runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception
{
QFunctionMetaData qFunctionMetaData = new BasicETLCleanupSourceFilesFunction().defineFunctionMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addFunction(qFunctionMetaData);
QBackendStepMetaData backendStepMetaData = new BasicETLCleanupSourceFilesStep().defineStepMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
qInstance.addProcess(qProcessMetaData);
HashSet<String> filePathsSet = new HashSet<>(filePaths);
@ -193,22 +193,22 @@ public class BasicETLCleanupSourceFilesFunctionTest
// List<QRecord> records = filePaths.stream()
// .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList();
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(qFunctionMetaData.getName());
runFunctionRequest.setProcessName(qProcessMetaData.getName());
RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runBackendStepRequest.setStepName(backendStepMetaData.getName());
runBackendStepRequest.setProcessName(qProcessMetaData.getName());
// runFunctionRequest.setRecords(records);
runFunctionRequest.setSession(TestUtils.getMockSession());
runFunctionRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
runFunctionRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
runFunctionRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
runBackendStepRequest.setSession(TestUtils.getMockSession());
runBackendStepRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
runBackendStepRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
runBackendStepRequest.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
for(Map.Entry<String, String> entry : values.entrySet())
{
runFunctionRequest.addValue(entry.getKey(), entry.getValue());
runBackendStepRequest.addValue(entry.getKey(), entry.getValue());
}
RunFunctionAction runFunctionAction = new RunFunctionAction();
return (runFunctionAction.execute(runFunctionRequest));
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
return (runFunctionAction.execute(runBackendStepRequest));
}

115
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStepTest.java Normal file
View File

@ -0,0 +1,115 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
import java.util.Arrays;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for BasicETLCollectSourceFileNamesFunction
*******************************************************************************/
class BasicETLCollectSourceFileNamesStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testOneFile() throws Exception
{
String file = "/tmp/test1.csv";
String result = runTest(file);
assertEquals(file, result);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testTwoFiles() throws Exception
{
String file1 = "/tmp/test1.csv";
String file2 = "/tmp/test2.csv";
String result = runTest(file1, file2);
//////////////////////////////////////////////////////////////////////
// the names go into a set, so they can come out in either order... //
//////////////////////////////////////////////////////////////////////
assertTrue(result.equals(file1 + "," + file2) || result.equals((file2 + "," + file1)));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDuplicatedFile() throws Exception
{
String file = "/tmp/test1.csv";
String result = runTest(file, file);
assertEquals(file, result);
}
/*******************************************************************************
**
*******************************************************************************/
private String runTest(String... fileNames) throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QBackendStepMetaData backendStepMetaData = new BasicETLCollectSourceFileNamesStep().defineStepMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
qInstance.addProcess(qProcessMetaData);
List<QRecord> records = Arrays.stream(fileNames).map(fileName ->
new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fileName)).toList();
RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runBackendStepRequest.setSession(TestUtils.getMockSession());
runBackendStepRequest.setStepName(backendStepMetaData.getName());
runBackendStepRequest.setProcessName(qProcessMetaData.getName());
runBackendStepRequest.setRecords(records);
RunBackendStepAction runBackendStepAction = new RunBackendStepAction();
RunBackendStepResult result = runBackendStepAction.execute(runBackendStepRequest);
return ((String) result.getValues().get(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS));
}
}

68
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java
View File

@ -25,16 +25,16 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi
import java.util.List;
import java.util.stream.Collectors;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -80,13 +80,13 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
QTableMetaData archiveTable = defineTable(qInstance, "archive", archiveBackend, "archive", "*/l3/*.csv");
QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QBackendStepMetaData step = process.getBackendStep(FilesystemSyncStep.STEP_NAME);
qInstance.addProcess(process);
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
///////////////////////////
// write some test files //
@ -99,17 +99,17 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
printTableListing(archiveBackend, archiveTable);
printTableListing(processingBackend, processingTable);
//////////////////////
// run the function //
//////////////////////
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(function.getName());
runFunctionRequest.setProcessName(process.getName());
runFunctionRequest.setSession(TestUtils.getMockSession());
//////////////////
// run the step //
//////////////////
RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runBackendStepRequest.setStepName(step.getName());
runBackendStepRequest.setProcessName(process.getName());
runBackendStepRequest.setSession(TestUtils.getMockSession());
RunFunctionAction runFunctionAction = new RunFunctionAction();
RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
System.out.println(runFunctionResult);
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
// System.out.println(runBackendStepResult);
printTableListing(sourceBackend, sourceTable);
printTableListing(archiveBackend, archiveTable);
@ -143,13 +143,13 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
QTableMetaData archiveTable = defineTable(qInstance, "archive", localBackend, "archive", "*/l3/*.csv");
QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QBackendStepMetaData step = process.getBackendStep(FilesystemSyncStep.STEP_NAME);
qInstance.addProcess(process);
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
///////////////////////////
// write some test files //
@ -162,17 +162,17 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
printTableListing(localBackend, archiveTable);
printTableListing(localBackend, processingTable);
//////////////////////
// run the function //
//////////////////////
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(function.getName());
runFunctionRequest.setProcessName(process.getName());
runFunctionRequest.setSession(TestUtils.getMockSession());
//////////////////
// run the step //
//////////////////
RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runBackendStepRequest.setStepName(step.getName());
runBackendStepRequest.setProcessName(process.getName());
runBackendStepRequest.setSession(TestUtils.getMockSession());
RunFunctionAction runFunctionAction = new RunFunctionAction();
RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
System.out.println(runFunctionResult);
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
// System.out.println(runBackendStepResult);
printTableListing(vendorBackend, sourceTable);
printTableListing(localBackend, archiveTable);

42
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java
View File

@ -24,14 +24,14 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi
import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
@ -54,16 +54,16 @@ class FilesystemSyncProcessTest
{
TestUtils.cleanInstanceFiles();
QTableMetaData sourceTable = defineTable("source");
QTableMetaData archiveTable = defineTable("archive");
QTableMetaData processingTable = defineTable("processing");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
QTableMetaData sourceTable = defineTable("source");
QTableMetaData archiveTable = defineTable("archive");
QTableMetaData processingTable = defineTable("processing");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QBackendStepMetaData step = (QBackendStepMetaData) process.getStep(FilesystemSyncStep.STEP_NAME);
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
// function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1);
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
// step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1);
QInstance qInstance = TestUtils.defineInstance();
qInstance.addTable(sourceTable);
@ -81,16 +81,16 @@ class FilesystemSyncProcessTest
writeTestFile(basePath, archiveTable, "2.txt", "x");
//////////////////////
// run the function //
// run the step //
//////////////////////
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(function.getName());
runFunctionRequest.setProcessName(process.getName());
runFunctionRequest.setSession(TestUtils.getMockSession());
RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runBackendStepRequest.setStepName(step.getName());
runBackendStepRequest.setProcessName(process.getName());
runBackendStepRequest.setSession(TestUtils.getMockSession());
RunFunctionAction runFunctionAction = new RunFunctionAction();
RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
System.out.println(runFunctionResult);
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
// System.out.println(runBackendStepResult);
}

67
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3ActionTest.java Normal file
View File

@ -0,0 +1,67 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
/*******************************************************************************
** Unit test for AbstractS3Action
*******************************************************************************/
class AbstractS3ActionTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBuildAmazonS3ClientFromBackendMetaData()
{
String regionName = Regions.AP_SOUTHEAST_3.getName();
S3BackendMetaData s3BackendMetaData = new S3BackendMetaData()
.withAccessKey("Not a real access key")
.withSecretKey("Also not a real key")
.withRegion(regionName);
AmazonS3 amazonS3 = new AbstractS3Action().buildAmazonS3ClientFromBackendMetaData(s3BackendMetaData);
assertEquals(regionName, amazonS3.getRegionName());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBuildAmazonS3ClientFromBackendMetaDataWrongType()
{
assertThrows(IllegalArgumentException.class, () ->
{
new AbstractS3Action().buildAmazonS3ClientFromBackendMetaData(new QBackendMetaData());
});
}
}

2
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java
View File

@ -52,7 +52,7 @@ class S3BackendMetaDataTest
System.out.println(JsonUtils.prettyPrint(json));
System.out.println(json);
String expectToContain = """
{"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","secretKey":null,"accessKey":null,"backendType":"s3","name":"s3","region":null}""";
{"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","backendType":"s3","name":"s3"}""";
assertTrue(json.contains(expectToContain));
}