QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release/0.2.0'

Browse Source
This commit is contained in:
darin.kelkhoff
2022-07-28 09:17:41 -05:00
parent 09a26e336a 0d5097564f
commit e9cb5c386c
32 changed files with 217 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.circleci/config.yml
View File

@ -37,6 +37,8 @@ commands:
name: Run Maven
command: |
mvn -s .circleci/mvn-settings.xml << parameters.maven_subcommand >>
- store_artifacts:
path: target/site/jacoco
- run:
name: Save test results
command: |

2
.gitignore vendored
View File

@ -1,5 +1,6 @@
target/
*.iml
.env
#############################################
## Original contents from github template: ##
@ -27,3 +28,4 @@ target/
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*
.DS_Store

6
pom.xml
View File

@ -25,7 +25,7 @@
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-backend-module-filesystem</artifactId>
<version>0.1.0</version>
<version>0.2.0</version>
<scm>
<connection>scm:git:git@github.com:Kingsrook/qqq-backend-module-rdbms.git</connection>
@ -53,14 +53,14 @@
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-backend-core</artifactId>
<version>0.1.0</version>
<version>0.2.0</version>
</dependency>
<!-- 3rd party deps specifically for this module -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.243</version>
<version>1.12.261</version>
</dependency>
<dependency>
<groupId>cloud.localstack</groupId>

29
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
View File

@ -25,21 +25,20 @@ package com.kingsrook.qqq.backend.module.filesystem.base.actions;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
@ -180,19 +179,17 @@ public abstract class AbstractBaseFilesystemAction<FILE>
/*******************************************************************************
** Generic implementation of the execute method from the QueryInterface
*******************************************************************************/
public QueryResult executeQuery(QueryRequest queryRequest) throws QException
public QueryOutput executeQuery(QueryInput queryInput) throws QException
{
preAction(queryRequest.getBackend());
preAction(queryInput.getBackend());
try
{
QueryResult rs = new QueryResult();
List<QRecord> records = new ArrayList<>();
rs.setRecords(records);
QueryOutput queryOutput = new QueryOutput(queryInput);
QTableMetaData table = queryRequest.getTable();
QTableMetaData table = queryInput.getTable();
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
List<FILE> files = listFiles(table, queryRequest.getBackend());
List<FILE> files = listFiles(table, queryInput.getBackend());
for(FILE file : files)
{
@ -207,7 +204,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
records.addAll(recordsInFile);
queryOutput.addRecords(recordsInFile);
break;
}
case JSON:
@ -218,7 +215,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
records.addAll(recordsInFile);
queryOutput.addRecords(recordsInFile);
break;
}
default:
@ -228,7 +225,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
}
}
return rs;
return queryOutput;
}
catch(Exception e)
{

2
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java
View File

@ -22,7 +22,7 @@
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
/*******************************************************************************

12
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java
View File

@ -23,13 +23,13 @@ package com.kingsrook.qqq.backend.module.filesystem.local;
import java.io.File;
import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;

2
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java
View File

@ -31,7 +31,7 @@ import java.util.Collections;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import org.apache.commons.io.FileUtils;

8
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java
View File

@ -22,10 +22,10 @@
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
import org.apache.commons.lang.NotImplementedException;
@ -38,7 +38,7 @@ public class FilesystemDeleteAction implements DeleteInterface
/*******************************************************************************
**
*******************************************************************************/
public DeleteResult execute(DeleteRequest deleteRequest) throws QException
public DeleteOutput execute(DeleteInput deleteInput) throws QException
{
throw new NotImplementedException("Filesystem delete not implemented");
/*

8
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java
View File

@ -22,10 +22,10 @@
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import org.apache.commons.lang.NotImplementedException;
@ -38,7 +38,7 @@ public class FilesystemInsertAction implements InsertInterface
/*******************************************************************************
**
*******************************************************************************/
public InsertResult execute(InsertRequest insertRequest) throws QException
public InsertOutput execute(InsertInput insertInput) throws QException
{
throw new NotImplementedException("Filesystem insert not implemented");
/*

10
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryAction.java
View File

@ -22,10 +22,10 @@
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
/*******************************************************************************
@ -37,9 +37,9 @@ public class FilesystemQueryAction extends AbstractFilesystemAction implements Q
/*******************************************************************************
**
*******************************************************************************/
public QueryResult execute(QueryRequest queryRequest) throws QException
public QueryOutput execute(QueryInput queryInput) throws QException
{
return executeQuery(queryRequest);
return executeQuery(queryInput);
}
}

8
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java
View File

@ -22,10 +22,10 @@
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
import org.apache.commons.lang.NotImplementedException;
@ -38,7 +38,7 @@ public class FilesystemUpdateAction implements UpdateInterface
/*******************************************************************************
**
*******************************************************************************/
public UpdateResult execute(UpdateRequest updateRequest) throws QException
public UpdateOutput execute(UpdateInput updateInput) throws QException
{
throw new NotImplementedException("Filesystem update not implemented");
/*

40
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java
View File

@ -23,21 +23,21 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et
import java.io.File;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
@ -67,11 +67,11 @@ public class BasicETLCleanupSourceFilesStep implements BackendStep
** Execute the step - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
String sourceTableName = runBackendStepRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
QTableMetaData table = runBackendStepRequest.getInstance().getTable(sourceTableName);
QBackendMetaData backend = runBackendStepRequest.getInstance().getBackendForTable(sourceTableName);
String sourceTableName = runBackendStepInput.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
QTableMetaData table = runBackendStepInput.getInstance().getTable(sourceTableName);
QBackendMetaData backend = runBackendStepInput.getInstance().getBackendForTable(sourceTableName);
QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend);
if(!(module instanceof FilesystemBackendModuleInterface filesystemModule))
@ -81,7 +81,7 @@ public class BasicETLCleanupSourceFilesStep implements BackendStep
AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase();
actionBase.preAction(backend);
String sourceFilePaths = runBackendStepRequest.getValueString(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS);
String sourceFilePaths = runBackendStepInput.getValueString(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS);
if(!StringUtils.hasContent(sourceFilePaths))
{
LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS + "]");
@ -91,21 +91,21 @@ public class BasicETLCleanupSourceFilesStep implements BackendStep
String[] sourceFiles = sourceFilePaths.split(",");
for(String sourceFile : sourceFiles)
{
String moveOrDelete = runBackendStepRequest.getValueString(FIELD_MOVE_OR_DELETE);
String moveOrDelete = runBackendStepInput.getValueString(FIELD_MOVE_OR_DELETE);
if(VALUE_DELETE.equals(moveOrDelete))
{
actionBase.deleteFile(runBackendStepRequest.getInstance(), table, sourceFile);
actionBase.deleteFile(runBackendStepInput.getInstance(), table, sourceFile);
}
else if(VALUE_MOVE.equals(moveOrDelete))
{
String destinationForMoves = runBackendStepRequest.getValueString(FIELD_DESTINATION_FOR_MOVES);
String destinationForMoves = runBackendStepInput.getValueString(FIELD_DESTINATION_FOR_MOVES);
if(!StringUtils.hasContent(destinationForMoves))
{
throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value."));
}
String filePathWithoutBase = actionBase.stripBackendAndTableBasePathsFromFileName(sourceFile, backend, table);
String destinationPath = destinationForMoves + File.separator + filePathWithoutBase;
actionBase.moveFile(runBackendStepRequest.getInstance(), table, sourceFile, destinationPath);
actionBase.moveFile(runBackendStepInput.getInstance(), table, sourceFile, destinationPath);
}
else
{

22
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java
View File

@ -24,15 +24,15 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -57,12 +57,12 @@ public class BasicETLCollectSourceFileNamesStep implements BackendStep
** Execute the step - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
Set<String> sourceFiles = runBackendStepRequest.getRecords().stream()
Set<String> sourceFiles = runBackendStepInput.getRecords().stream()
.map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))
.collect(Collectors.toSet());
runBackendStepResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles));
runBackendStepOutput.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles));
}

10
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java
View File

@ -22,11 +22,11 @@
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;

26
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java
View File

@ -28,13 +28,13 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import org.apache.logging.log4j.LogManager;
@ -59,30 +59,30 @@ public class FilesystemSyncStep implements BackendStep
** Execute the step - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
QTableMetaData sourceTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
QTableMetaData archiveTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
QTableMetaData processingTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
QTableMetaData sourceTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
QTableMetaData archiveTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
QTableMetaData processingTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
QBackendMetaData sourceBackend = runBackendStepRequest.getInstance().getBackendForTable(sourceTable.getName());
QBackendMetaData sourceBackend = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName());
FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase();
sourceActionBase.preAction(sourceBackend);
Map<String, Object> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
QBackendMetaData archiveBackend = runBackendStepRequest.getInstance().getBackendForTable(archiveTable.getName());
QBackendMetaData archiveBackend = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName());
FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase();
archiveActionBase.preAction(archiveBackend);
Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet();
QBackendMetaData processingBackend = runBackendStepRequest.getInstance().getBackendForTable(processingTable.getName());
QBackendMetaData processingBackend = runBackendStepInput.getInstance().getBackendForTable(processingTable.getName());
FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase();
processingActionBase.preAction(processingBackend);
Integer maxFilesToSync = runBackendStepRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
Integer maxFilesToSync = runBackendStepInput.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
int syncedFileCount = 0;
for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet())
{

12
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java
View File

@ -23,13 +23,13 @@ package com.kingsrook.qqq.backend.module.filesystem.s3;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;

2
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java
View File

@ -32,7 +32,7 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;

8
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java
View File

@ -22,10 +22,10 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
import org.apache.commons.lang.NotImplementedException;
@ -38,7 +38,7 @@ public class S3DeleteAction implements DeleteInterface
/*******************************************************************************
**
*******************************************************************************/
public DeleteResult execute(DeleteRequest deleteRequest) throws QException
public DeleteOutput execute(DeleteInput deleteInput) throws QException
{
throw new NotImplementedException("S3 delete not implemented");
/*

8
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java
View File

@ -22,10 +22,10 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import org.apache.commons.lang.NotImplementedException;
@ -38,7 +38,7 @@ public class S3InsertAction implements InsertInterface
/*******************************************************************************
**
*******************************************************************************/
public InsertResult execute(InsertRequest insertRequest) throws QException
public InsertOutput execute(InsertInput insertInput) throws QException
{
throw new NotImplementedException("S3 insert not implemented");
/*

10
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryAction.java
View File

@ -22,10 +22,10 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
/*******************************************************************************
@ -37,9 +37,9 @@ public class S3QueryAction extends AbstractS3Action implements QueryInterface
/*******************************************************************************
**
*******************************************************************************/
public QueryResult execute(QueryRequest queryRequest) throws QException
public QueryOutput execute(QueryInput queryInput) throws QException
{
return (super.executeQuery(queryRequest));
return (super.executeQuery(queryInput));
}
}

8
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java
View File

@ -22,10 +22,10 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
import org.apache.commons.lang.NotImplementedException;
@ -38,7 +38,7 @@ public class S3UpdateAction implements UpdateInterface
/*******************************************************************************
**
*******************************************************************************/
public UpdateResult execute(UpdateRequest updateRequest) throws QException
public UpdateOutput execute(UpdateInput updateInput) throws QException
{
throw new NotImplementedException("S3 update not implemented");
/*

17
src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -26,13 +26,14 @@ import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.mock.MockAuthenticationModule;
import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationModule;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
@ -135,7 +136,7 @@ public class TestUtils
{
return new QAuthenticationMetaData()
.withName("mock")
.withType("mock");
.withType(QAuthenticationType.MOCK);
}
@ -219,9 +220,9 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QSession getMockSession()
public static QSession getMockSession() throws QInstanceValidationException
{
MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule();
return (mockAuthenticationModule.createSession(null));
return (mockAuthenticationModule.createSession(defineInstance(), null));
}
}

2
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java
View File

@ -26,7 +26,7 @@ import java.io.File;
import java.io.IOException;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;

38
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java
View File

@ -24,13 +24,13 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
@ -50,12 +50,12 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
@Test
public void testQuery1() throws QException
{
QueryRequest queryRequest = new QueryRequest();
queryRequest.setInstance(TestUtils.defineInstance());
queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows");
Assertions.assertTrue(queryResult.getRecords().stream()
QueryInput queryInput = new QueryInput();
queryInput.setInstance(TestUtils.defineInstance());
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
Assertions.assertTrue(queryOutput.getRecords().stream()
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
"All records should have a full-path in their backend details, matching the test folder name");
}
@ -68,8 +68,8 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
@Test
public void testQueryWithFileCustomizer() throws QException
{
QueryRequest queryRequest = new QueryRequest();
QInstance instance = TestUtils.defineInstance();
QueryInput queryInput = new QueryInput();
QInstance instance = TestUtils.defineInstance();
QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
table.withCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ, new QCodeReference()
@ -77,12 +77,12 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.CUSTOMIZER));
queryRequest.setInstance(instance);
queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows");
queryInput.setInstance(instance);
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
Assertions.assertTrue(
queryResult.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")),
queryOutput.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")),
"All records should have their email addresses up-shifted.");
}

5
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java
View File

@ -70,8 +70,7 @@ class FilesystemBackendMetaDataTest
String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance);
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
assertThat(deserialized).usingRecursiveComparison()
.ignoringFields("hasBeenValidated") // note, this field is @JsonIgnore
.isEqualTo(qInstance);
assertThat(deserialized.getBackends()).usingRecursiveComparison()
.isEqualTo(qInstance.getBackends());
}
}

34
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java
View File

@ -27,9 +27,9 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
@ -136,13 +136,13 @@ public class BasicETLCleanupSourceFilesStepTest
*******************************************************************************/
private void testDelete(QInstance qInstance, List<String> filePaths) throws Exception
{
RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of(
RunBackendStepOutput runBackendStepOutput = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesStep.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesStep.VALUE_DELETE,
// todo - even though this field isn't needed, since we gave a value of "delete"
// the RunFunctionAction considers any missing input to be an error...
BasicETLCleanupSourceFilesStep.FIELD_DESTINATION_FOR_MOVES, ""));
assertNull(runBackendStepResult.getException());
assertNull(runBackendStepOutput.getException());
for(String filePath : filePaths)
{
assertFalse(new File(filePath).exists(), "File should have been deleted.");
@ -157,11 +157,11 @@ public class BasicETLCleanupSourceFilesStepTest
private void testMove(QInstance qInstance, List<String> filePaths) throws Exception
{
String trashDir = File.separator + "tmp" + File.separator + "trash";
RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of(
RunBackendStepOutput runBackendStepOutput = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesStep.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesStep.VALUE_MOVE,
BasicETLCleanupSourceFilesStep.FIELD_DESTINATION_FOR_MOVES, trashDir));
assertNull(runBackendStepResult.getException());
assertNull(runBackendStepOutput.getException());
for(String filePath : filePaths)
{
@ -177,7 +177,7 @@ public class BasicETLCleanupSourceFilesStepTest
/*******************************************************************************
**
*******************************************************************************/
private RunBackendStepResult runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception
private RunBackendStepOutput runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception
{
QBackendStepMetaData backendStepMetaData = new BasicETLCleanupSourceFilesStep().defineStepMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
@ -193,22 +193,22 @@ public class BasicETLCleanupSourceFilesStepTest
// List<QRecord> records = filePaths.stream()
// .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList();
RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runBackendStepRequest.setStepName(backendStepMetaData.getName());
runBackendStepRequest.setProcessName(qProcessMetaData.getName());
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(qInstance);
runBackendStepInput.setStepName(backendStepMetaData.getName());
runBackendStepInput.setProcessName(qProcessMetaData.getName());
// runFunctionRequest.setRecords(records);
runBackendStepRequest.setSession(TestUtils.getMockSession());
runBackendStepRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
runBackendStepRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
runBackendStepRequest.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
runBackendStepInput.setSession(TestUtils.getMockSession());
runBackendStepInput.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
runBackendStepInput.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
runBackendStepInput.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
for(Map.Entry<String, String> entry : values.entrySet())
{
runBackendStepRequest.addValue(entry.getKey(), entry.getValue());
runBackendStepInput.addValue(entry.getKey(), entry.getValue());
}
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
return (runFunctionAction.execute(runBackendStepRequest));
return (runFunctionAction.execute(runBackendStepInput));
}

18
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStepTest.java
View File

@ -24,9 +24,9 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et
import java.util.Arrays;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
@ -101,14 +101,14 @@ class BasicETLCollectSourceFileNamesStepTest
List<QRecord> records = Arrays.stream(fileNames).map(fileName ->
new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fileName)).toList();
RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runBackendStepRequest.setSession(TestUtils.getMockSession());
runBackendStepRequest.setStepName(backendStepMetaData.getName());
runBackendStepRequest.setProcessName(qProcessMetaData.getName());
runBackendStepRequest.setRecords(records);
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(qInstance);
runBackendStepInput.setSession(TestUtils.getMockSession());
runBackendStepInput.setStepName(backendStepMetaData.getName());
runBackendStepInput.setProcessName(qProcessMetaData.getName());
runBackendStepInput.setRecords(records);
RunBackendStepAction runBackendStepAction = new RunBackendStepAction();
RunBackendStepResult result = runBackendStepAction.execute(runBackendStepRequest);
RunBackendStepOutput result = runBackendStepAction.execute(runBackendStepInput);
return ((String) result.getValues().get(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS));
}

34
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java
View File

@ -25,18 +25,18 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi
import java.util.List;
import java.util.stream.Collectors;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
@ -102,13 +102,13 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
//////////////////
// run the step //
//////////////////
RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runBackendStepRequest.setStepName(step.getName());
runBackendStepRequest.setProcessName(process.getName());
runBackendStepRequest.setSession(TestUtils.getMockSession());
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(qInstance);
runBackendStepInput.setStepName(step.getName());
runBackendStepInput.setProcessName(process.getName());
runBackendStepInput.setSession(TestUtils.getMockSession());
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
RunBackendStepOutput runBackendStepOutput = runFunctionAction.execute(runBackendStepInput);
// System.out.println(runBackendStepResult);
printTableListing(sourceBackend, sourceTable);
@ -165,13 +165,13 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
//////////////////
// run the step //
//////////////////
RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runBackendStepRequest.setStepName(step.getName());
runBackendStepRequest.setProcessName(process.getName());
runBackendStepRequest.setSession(TestUtils.getMockSession());
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(qInstance);
runBackendStepInput.setStepName(step.getName());
runBackendStepInput.setProcessName(process.getName());
runBackendStepInput.setSession(TestUtils.getMockSession());
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
RunBackendStepOutput runBackendStepOutput = runFunctionAction.execute(runBackendStepInput);
// System.out.println(runBackendStepResult);
printTableListing(vendorBackend, sourceTable);

22
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java
View File

@ -24,13 +24,13 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi
import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -83,13 +83,13 @@ class FilesystemSyncProcessTest
//////////////////////
// run the step //
//////////////////////
RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
runBackendStepRequest.setStepName(step.getName());
runBackendStepRequest.setProcessName(process.getName());
runBackendStepRequest.setSession(TestUtils.getMockSession());
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(qInstance);
runBackendStepInput.setStepName(step.getName());
runBackendStepInput.setProcessName(process.getName());
runBackendStepInput.setSession(TestUtils.getMockSession());
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
RunBackendStepOutput runBackendStepOutput = runFunctionAction.execute(runBackendStepInput);
// System.out.println(runBackendStepResult);
}

2
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java
View File

@ -26,7 +26,7 @@ import java.util.List;
import java.util.UUID;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;

22
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java
View File

@ -24,8 +24,8 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
@ -45,12 +45,12 @@ public class S3QueryActionTest extends BaseS3Test
@Test
public void testQuery1() throws QException
{
QueryRequest queryRequest = initQueryRequest();
QueryInput queryInput = initQueryRequest();
S3QueryAction s3QueryAction = new S3QueryAction();
s3QueryAction.setS3Utils(getS3Utils());
QueryResult queryResult = s3QueryAction.execute(queryRequest);
Assertions.assertEquals(5, queryResult.getRecords().size(), "Expected # of rows from unfiltered query");
Assertions.assertTrue(queryResult.getRecords().stream()
QueryOutput queryOutput = s3QueryAction.execute(queryInput);
Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
Assertions.assertTrue(queryOutput.getRecords().stream()
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseS3Test.TEST_FOLDER)),
"All records should have a full-path in their backend details, matching the test folder name");
}
@ -60,12 +60,12 @@ public class S3QueryActionTest extends BaseS3Test
/*******************************************************************************
**
*******************************************************************************/
private QueryRequest initQueryRequest() throws QInstanceValidationException
private QueryInput initQueryRequest() throws QInstanceValidationException
{
QueryRequest queryRequest = new QueryRequest();
queryRequest.setInstance(TestUtils.defineInstance());
queryRequest.setTableName(TestUtils.defineS3CSVPersonTable().getName());
return queryRequest;
QueryInput queryInput = new QueryInput();
queryInput.setInstance(TestUtils.defineInstance());
queryInput.setTableName(TestUtils.defineS3CSVPersonTable().getName());
return queryInput;
}
}

5
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java
View File

@ -70,8 +70,7 @@ class S3BackendMetaDataTest
String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance);
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
assertThat(deserialized).usingRecursiveComparison()
.ignoringFields("hasBeenValidated") // note, this field is @JsonIgnore
.isEqualTo(qInstance);
assertThat(deserialized.getBackends()).usingRecursiveComparison()
.isEqualTo(qInstance.getBackends());
}
}