Merge pull request #2 from Kingsrook/feature/QQQ-21-processes-in-ui

Feature/qqq 21 processes in UI

.circleci/config.yml

@@ -57,7 +57,7 @@ jobs:
       - localstack/startup
       - install_java17
       - run_maven:
-          maven_subcommand: test
+          maven_subcommand: verify
       - slack/notify:
           event: fail
 

checkstyle.xml

@@ -181,8 +181,8 @@
       </module>
       -->
       <module name="OverloadMethodsDeclarationOrder"/>
-      <module name="VariableDeclarationUsageDistance"/>
       <!--
+      <module name="VariableDeclarationUsageDistance"/>
       <module name="CustomImportOrder">
          <property name="sortImportsInGroupAlphabetically" value="true"/>
          <property name="separateLineBetweenGroups" value="true"/>

pom.xml

@@ -44,6 +44,8 @@
       <maven.compiler.target>17</maven.compiler.target>
       <maven.compiler.showDeprecation>true</maven.compiler.showDeprecation>
       <maven.compiler.showWarnings>true</maven.compiler.showWarnings>
+      <coverage.haltOnFailure>true</coverage.haltOnFailure>
+      <coverage.instructionCoveredRatioMinimum>0.80</coverage.instructionCoveredRatioMinimum>
    </properties>
 
    <dependencies>
@@ -51,7 +53,7 @@
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
          <artifactId>qqq-backend-core</artifactId>
-         <version>0.0.0</version>
+         <version>0.1.0-20220708.195335-5</version>
       </dependency>
 
       <!-- 3rd party deps specifically for this module -->
@@ -115,6 +117,9 @@
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-surefire-plugin</artifactId>
             <version>3.0.0-M5</version>
+            <configuration>
+               <argLine>@{jaCoCoArgLine}</argLine>
+            </configuration>
          </plugin>
          <plugin>
             <groupId>org.apache.maven.plugins</groupId>
@@ -164,7 +169,84 @@
                <versionDigitToIncrement>1</versionDigitToIncrement> <!-- In general, we update the minor -->
             </configuration>
          </plugin>
-
+         <plugin>
+            <groupId>org.jacoco</groupId>
+            <artifactId>jacoco-maven-plugin</artifactId>
+            <version>0.8.8</version>
+            <executions>
+               <execution>
+                  <id>pre-unit-test</id>
+                  <goals>
+                     <goal>prepare-agent</goal>
+                  </goals>
+                  <configuration>
+                     <propertyName>jaCoCoArgLine</propertyName>
+                  </configuration>
+               </execution>
+               <execution>
+                  <id>unit-test-check</id>
+                  <goals>
+                     <goal>check</goal>
+                  </goals>
+                  <configuration>
+                     <!-- Gives us the ability to pass a parameter to not fail due to coverage E.g. -Dcoverage.haltOnFailure=false -->
+                     <haltOnFailure>${coverage.haltOnFailure}</haltOnFailure>
+                     <rules>
+                        <rule>
+                           <element>BUNDLE</element>
+                           <limits>
+                              <limit>
+                                 <counter>INSTRUCTION</counter>
+                                 <value>COVEREDRATIO</value>
+                                 <minimum>${coverage.instructionCoveredRatioMinimum}</minimum>
+                              </limit>
+                           </limits>
+                        </rule>
+                     </rules>
+                  </configuration>
+               </execution>
+               <execution>
+                  <id>post-unit-test</id>
+                  <phase>verify</phase>
+                  <goals>
+                     <goal>report</goal>
+                  </goals>
+               </execution>
+            </executions>
+         </plugin>
+         <plugin>
+            <artifactId>exec-maven-plugin</artifactId>
+            <groupId>org.codehaus.mojo</groupId>
+            <version>3.0.0</version>
+            <executions>
+               <execution>
+                  <id>test-coverage-summary</id>
+                  <phase>verify</phase>
+                  <goals>
+                     <goal>exec</goal>
+                  </goals>
+                  <configuration>
+                     <executable>sh</executable>
+                     <arguments>
+                        <argument>-c</argument>
+                        <argument>
+                           <![CDATA[
+echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
+xpath -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
+echo
+echo "Jacoco coverage summary report:"
+echo "   See also target/site/jacoco/index.html"
+echo "   and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
+echo "------------------------------------------------------------"
+paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
+rm /tmp/$$.headers /tmp/$$.values
+                           ]]>
+                        </argument>
+                     </arguments>
+                  </configuration>
+               </execution>
+            </executions>
+         </plugin>
       </plugins>
    </build>
 

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java

@@ -31,6 +31,8 @@ import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFile
  *******************************************************************************/
 public interface FilesystemBackendModuleInterface<FILE>
 {
+   String CUSTOMIZER_FILE_POST_FILE_READ = "postFileRead";
+
    /*******************************************************************************
     ** For filesystem backends, get the module-specific action base-class, that helps
     ** with functions like listing and deleting files.

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java

@@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
@@ -269,7 +270,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
     *******************************************************************************/
    private String customizeFileContentsAfterReading(QTableMetaData table, String fileContents) throws QException
    {
-      Optional<QCodeReference> optionalCustomizer = table.getCustomizer("postFileRead");
+      Optional<QCodeReference> optionalCustomizer = table.getCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ);
       if(optionalCustomizer.isEmpty())
       {
          return (fileContents);

src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java

@@ -24,9 +24,9 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et
 
 import java.io.File;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
+import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
@@ -34,8 +34,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
 import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
@@ -47,65 +47,65 @@ import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
- ** Function body for performing the Cleanup step of a basic ETL process - e.g.,
+ ** BackendStep for performing the Cleanup step of a basic ETL process - e.g.,
  ** after the loading, delete or move the processed file(s).
  *******************************************************************************/
-public class BasicETLCleanupSourceFilesFunction implements FunctionBody
+public class BasicETLCleanupSourceFilesStep implements BackendStep
 {
-   private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesFunction.class);
+   private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesStep.class);
 
    public static final String FIELD_MOVE_OR_DELETE        = "moveOrDelete";
    public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves";
 
    public static final String VALUE_MOVE    = "move";
    public static final String VALUE_DELETE = "delete";
-   public static final String FUNCTION_NAME = "cleanupSourceFiles";
+   public static final String STEP_NAME    = "cleanupSourceFiles";
 
 
 
    /*******************************************************************************
-    ** Execute the function - using the request as input, and the result as output.
+    ** Execute the step - using the request as input, and the result as output.
     *******************************************************************************/
    @Override
-   public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
+   public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
    {
-      String                  sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
+      String                  sourceTableName = runBackendStepRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
-      QTableMetaData          table           = runFunctionRequest.getInstance().getTable(sourceTableName);
+      QTableMetaData          table           = runBackendStepRequest.getInstance().getTable(sourceTableName);
-      QBackendMetaData        backend         = runFunctionRequest.getInstance().getBackendForTable(sourceTableName);
+      QBackendMetaData        backend         = runBackendStepRequest.getInstance().getBackendForTable(sourceTableName);
       QBackendModuleInterface module          = new QBackendModuleDispatcher().getQBackendModule(backend);
 
       if(!(module instanceof FilesystemBackendModuleInterface filesystemModule))
       {
-         throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function."));
+         throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this action."));
       }
       AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase();
       actionBase.preAction(backend);
 
-      String sourceFilePaths = runFunctionRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS);
+      String sourceFilePaths = runBackendStepRequest.getValueString(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS);
       if(!StringUtils.hasContent(sourceFilePaths))
       {
-         LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS + "]");
+         LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS + "]");
          return;
       }
 
       String[] sourceFiles = sourceFilePaths.split(",");
       for(String sourceFile : sourceFiles)
       {
-         String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE);
+         String moveOrDelete = runBackendStepRequest.getValueString(FIELD_MOVE_OR_DELETE);
          if(VALUE_DELETE.equals(moveOrDelete))
          {
-            actionBase.deleteFile(runFunctionRequest.getInstance(), table, sourceFile);
+            actionBase.deleteFile(runBackendStepRequest.getInstance(), table, sourceFile);
          }
          else if(VALUE_MOVE.equals(moveOrDelete))
          {
-            String destinationForMoves = runFunctionRequest.getValueString(FIELD_DESTINATION_FOR_MOVES);
+            String destinationForMoves = runBackendStepRequest.getValueString(FIELD_DESTINATION_FOR_MOVES);
             if(!StringUtils.hasContent(destinationForMoves))
             {
                throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value."));
             }
             String filePathWithoutBase = actionBase.stripBackendAndTableBasePathsFromFileName(sourceFile, backend, table);
             String destinationPath     = destinationForMoves + File.separator + filePathWithoutBase;
-            actionBase.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath);
+            actionBase.moveFile(runBackendStepRequest.getInstance(), table, sourceFile, destinationPath);
          }
          else
          {
@@ -118,16 +118,16 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
 
 
    /*******************************************************************************
-    ** define the metaData that describes this function
+    ** define the metaData that describes this step
     *******************************************************************************/
-   public QFunctionMetaData defineFunctionMetaData()
+   public QBackendStepMetaData defineStepMetaData()
    {
-      return (new QFunctionMetaData()
+      return (new QBackendStepMetaData()
-         .withName(FUNCTION_NAME)
+         .withName(STEP_NAME)
          .withCode(new QCodeReference()
             .withName(this.getClass().getName())
             .withCodeType(QCodeType.JAVA)
-            .withCodeUsage(QCodeUsage.FUNCTION))
+            .withCodeUsage(QCodeUsage.BACKEND_STEP))
          .withInputData(new QFunctionInputMetaData()
             .addField(new QFieldMetaData("moveOrDelete", QFieldType.STRING))
             .addField(new QFieldMetaData("destinationForMoves", QFieldType.STRING))));

src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java

@@ -25,59 +25,59 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et
 import java.util.Set;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
+import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
 import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
 import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 
 
 /*******************************************************************************
- ** Function body for collecting the file names that were discovered in the
+ ** BackendStep for collecting the file names that were discovered in the
  ** Extract step.  These will be lost during the transform, so we capture them here,
- ** so that our Clean function can move or delete them.
+ ** so that our Clean step can move or delete them.
  **
  ** TODO - need unit test!!
  *******************************************************************************/
-public class BasicETLCollectSourceFileNamesFunction implements FunctionBody
+public class BasicETLCollectSourceFileNamesStep implements BackendStep
 {
-   public static final String FUNCTION_NAME           = "collectSourceFileNames";
+   public static final String STEP_NAME               = "collectSourceFileNames";
    public static final String FIELD_SOURCE_FILE_PATHS = "sourceFilePaths";
 
 
 
    /*******************************************************************************
-    ** Execute the function - using the request as input, and the result as output.
+    ** Execute the step - using the request as input, and the result as output.
     *******************************************************************************/
    @Override
-   public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
+   public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
    {
-      Set<String> sourceFiles = runFunctionRequest.getRecords().stream()
+      Set<String> sourceFiles = runBackendStepRequest.getRecords().stream()
          .map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))
          .collect(Collectors.toSet());
-      runFunctionResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles));
+      runBackendStepResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles));
    }
 
 
 
    /*******************************************************************************
-    ** define the metaData that describes this function
+    ** define the metaData that describes this step
     *******************************************************************************/
-   public QFunctionMetaData defineFunctionMetaData()
+   public QBackendStepMetaData defineStepMetaData()
    {
-      return (new QFunctionMetaData()
+      return (new QBackendStepMetaData()
-         .withName(FUNCTION_NAME)
+         .withName(STEP_NAME)
          .withCode(new QCodeReference()
             .withName(this.getClass().getName())
             .withCodeType(QCodeType.JAVA)
-            .withCodeUsage(QCodeUsage.FUNCTION))
+            .withCodeUsage(QCodeUsage.BACKEND_STEP))
          .withOutputMetaData(new QFunctionOutputMetaData()
             .addField(new QFieldMetaData(FIELD_SOURCE_FILE_PATHS, QFieldType.STRING))));
    }

src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java

@@ -27,8 +27,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
 import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 
 
@@ -64,12 +64,12 @@ public class FilesystemSyncProcess
     *******************************************************************************/
    public QProcessMetaData defineProcessMetaData()
    {
-      QFunctionMetaData syncFunction = new QFunctionMetaData()
+      QBackendStepMetaData syncStep = new QBackendStepMetaData()
-         .withName(FilesystemSyncFunction.FUNCTION_NAME)
+         .withName(FilesystemSyncStep.STEP_NAME)
          .withCode(new QCodeReference()
-            .withName(FilesystemSyncFunction.class.getName())
+            .withName(FilesystemSyncStep.class.getName())
             .withCodeType(QCodeType.JAVA)
-            .withCodeUsage(QCodeUsage.FUNCTION))
+            .withCodeUsage(QCodeUsage.BACKEND_STEP))
          .withInputData(new QFunctionInputMetaData()
             .addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING))
             .addField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING))
@@ -78,6 +78,6 @@ public class FilesystemSyncProcess
 
       return new QProcessMetaData()
          .withName(PROCESS_NAME)
-         .addFunction(syncFunction);
+         .addStep(syncStep);
    }
 }

src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java

@@ -29,9 +29,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
+import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
@@ -42,47 +42,47 @@ import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
- ** Function body for collecting the file names that were discovered in the
+ ** BackendStep to sync two filesystem tables (copying the new files to a 3rd
- ** Extract step.  These will be lost during the transform, so we capture them here,
+ ** location as well...)
- ** so that our Clean function can move or delete them.
  **
  *******************************************************************************/
-public class FilesystemSyncFunction implements FunctionBody
+@SuppressWarnings("unchecked")
+public class FilesystemSyncStep implements BackendStep
 {
-   private static final Logger LOG = LogManager.getLogger(FilesystemSyncFunction.class);
+   private static final Logger LOG = LogManager.getLogger(FilesystemSyncStep.class);
 
-   public static final String FUNCTION_NAME = "sync";
+   public static final String STEP_NAME = "sync";
 
 
 
    /*******************************************************************************
-    ** Execute the function - using the request as input, and the result as output.
+    ** Execute the step - using the request as input, and the result as output.
     *******************************************************************************/
    @Override
-   public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
+   public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
    {
-      QTableMetaData sourceTable     = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
+      QTableMetaData sourceTable     = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
-      QTableMetaData archiveTable    = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
+      QTableMetaData archiveTable    = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
-      QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
+      QTableMetaData processingTable = runBackendStepRequest.getInstance().getTable(runBackendStepRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
 
-      QBackendMetaData                 sourceBackend    = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName());
+      QBackendMetaData                 sourceBackend    = runBackendStepRequest.getInstance().getBackendForTable(sourceTable.getName());
       FilesystemBackendModuleInterface sourceModule     = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
       AbstractBaseFilesystemAction     sourceActionBase = sourceModule.getActionBase();
       sourceActionBase.preAction(sourceBackend);
       Map<String, Object> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
 
-      QBackendMetaData                 archiveBackend    = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName());
+      QBackendMetaData                 archiveBackend    = runBackendStepRequest.getInstance().getBackendForTable(archiveTable.getName());
       FilesystemBackendModuleInterface archiveModule     = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
       AbstractBaseFilesystemAction     archiveActionBase = archiveModule.getActionBase();
       archiveActionBase.preAction(archiveBackend);
       Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet();
 
-      QBackendMetaData                 processingBackend    = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName());
+      QBackendMetaData                 processingBackend    = runBackendStepRequest.getInstance().getBackendForTable(processingTable.getName());
       FilesystemBackendModuleInterface processingModule     = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
       AbstractBaseFilesystemAction     processingActionBase = processingModule.getActionBase();
       processingActionBase.preAction(processingBackend);
 
-      Integer maxFilesToSync = runFunctionRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
+      Integer maxFilesToSync = runBackendStepRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
       int syncedFileCount = 0;
       for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet())
       {

src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java

@@ -27,6 +27,7 @@ import java.io.InputStream;
 import java.util.List;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@@ -67,6 +68,17 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
          return;
       }
 
+      s3Utils = new S3Utils();
+      s3Utils.setAmazonS3(buildAmazonS3ClientFromBackendMetaData(backendMetaData));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected AmazonS3 buildAmazonS3ClientFromBackendMetaData(QBackendMetaData backendMetaData)
+   {
       S3BackendMetaData     s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendMetaData);
       AmazonS3ClientBuilder clientBuilder     = AmazonS3ClientBuilder.standard();
 
@@ -81,8 +93,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
          clientBuilder.setRegion(s3BackendMetaData.getRegion());
       }
 
-      s3Utils = new S3Utils();
+      return (clientBuilder.build());
-      s3Utils.setAmazonS3(clientBuilder.build());
    }
 
 

src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java

@@ -22,11 +22,17 @@
 package com.kingsrook.qqq.backend.module.filesystem.local.actions;
 
 
+import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
 import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
 import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -44,7 +50,9 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
    @Test
    public void testQuery1() throws QException
    {
-      QueryRequest queryRequest = initQueryRequest();
+      QueryRequest queryRequest = new QueryRequest();
+      queryRequest.setInstance(TestUtils.defineInstance());
+      queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
       QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
       Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows");
       Assertions.assertTrue(queryResult.getRecords().stream()
@@ -57,12 +65,36 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
    /*******************************************************************************
     **
     *******************************************************************************/
-   private QueryRequest initQueryRequest() throws QInstanceValidationException
+   @Test
+   public void testQueryWithFileCustomizer() throws QException
    {
       QueryRequest queryRequest = new QueryRequest();
-      queryRequest.setInstance(TestUtils.defineInstance());
+      QInstance    instance     = TestUtils.defineInstance();
+
+      QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
+      table.withCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ, new QCodeReference()
+         .withName(ValueUpshifter.class.getName())
+         .withCodeType(QCodeType.JAVA)
+         .withCodeUsage(QCodeUsage.CUSTOMIZER));
+
+      queryRequest.setInstance(instance);
       queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
-      return queryRequest;
+      QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
+      Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows");
+      Assertions.assertTrue(
+         queryResult.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")),
+         "All records should have their email addresses up-shifted.");
+   }
+
+
+
+   public static class ValueUpshifter implements Function<String, String>
+   {
+      @Override
+      public String apply(String s)
+      {
+         return (s.replaceAll("kingsrook.com", "KINGSROOK.COM"));
+      }
    }
 
 }

src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java

@@ -27,11 +27,11 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
-import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
+import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -48,7 +48,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 /*******************************************************************************
  ** Unit test for BasicETLCleanupSourceFilesFunction
  *******************************************************************************/
-public class BasicETLCleanupSourceFilesFunctionTest
+public class BasicETLCleanupSourceFilesStepTest
 {
 
    /*******************************************************************************
@@ -136,13 +136,13 @@ public class BasicETLCleanupSourceFilesFunctionTest
     *******************************************************************************/
    private void testDelete(QInstance qInstance, List<String> filePaths) throws Exception
    {
-      RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of(
+      RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of(
-         BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_DELETE,
+         BasicETLCleanupSourceFilesStep.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesStep.VALUE_DELETE,
          // todo - even though this field isn't needed, since we gave a value of "delete"
          //  the RunFunctionAction considers any missing input to be an error...
-         BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, ""));
+         BasicETLCleanupSourceFilesStep.FIELD_DESTINATION_FOR_MOVES, ""));
 
-      assertNull(runFunctionResult.getError());
+      assertNull(runBackendStepResult.getException());
       for(String filePath : filePaths)
       {
          assertFalse(new File(filePath).exists(), "File should have been deleted.");
@@ -157,11 +157,11 @@ public class BasicETLCleanupSourceFilesFunctionTest
    private void testMove(QInstance qInstance, List<String> filePaths) throws Exception
    {
       String trashDir = File.separator + "tmp" + File.separator + "trash";
-      RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of(
+      RunBackendStepResult runBackendStepResult = runFunction(qInstance, filePaths, Map.of(
-         BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_MOVE,
+         BasicETLCleanupSourceFilesStep.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesStep.VALUE_MOVE,
-         BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir));
+         BasicETLCleanupSourceFilesStep.FIELD_DESTINATION_FOR_MOVES, trashDir));
 
-      assertNull(runFunctionResult.getError());
+      assertNull(runBackendStepResult.getException());
 
       for(String filePath : filePaths)
       {
@@ -177,10 +177,10 @@ public class BasicETLCleanupSourceFilesFunctionTest
    /*******************************************************************************
     **
     *******************************************************************************/
-   private RunFunctionResult runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception
+   private RunBackendStepResult runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception
    {
-      QFunctionMetaData qFunctionMetaData = new BasicETLCleanupSourceFilesFunction().defineFunctionMetaData();
+      QBackendStepMetaData backendStepMetaData = new BasicETLCleanupSourceFilesStep().defineStepMetaData();
-      QProcessMetaData  qProcessMetaData  = new QProcessMetaData().withName("testScaffold").addFunction(qFunctionMetaData);
+      QProcessMetaData     qProcessMetaData  = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
       qInstance.addProcess(qProcessMetaData);
 
       HashSet<String> filePathsSet = new HashSet<>(filePaths);
@@ -193,22 +193,22 @@ public class BasicETLCleanupSourceFilesFunctionTest
       // List<QRecord> records = filePaths.stream()
       //    .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList();
 
-      RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
+      RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
-      runFunctionRequest.setFunctionName(qFunctionMetaData.getName());
+      runBackendStepRequest.setStepName(backendStepMetaData.getName());
-      runFunctionRequest.setProcessName(qProcessMetaData.getName());
+      runBackendStepRequest.setProcessName(qProcessMetaData.getName());
       // runFunctionRequest.setRecords(records);
-      runFunctionRequest.setSession(TestUtils.getMockSession());
+      runBackendStepRequest.setSession(TestUtils.getMockSession());
-      runFunctionRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
+      runBackendStepRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
-      runFunctionRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
+      runBackendStepRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
-      runFunctionRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
+      runBackendStepRequest.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
 
       for(Map.Entry<String, String> entry : values.entrySet())
       {
-         runFunctionRequest.addValue(entry.getKey(), entry.getValue());
+         runBackendStepRequest.addValue(entry.getKey(), entry.getValue());
       }
 
-      RunFunctionAction runFunctionAction = new RunFunctionAction();
+      RunBackendStepAction runFunctionAction = new RunBackendStepAction();
-      return (runFunctionAction.execute(runFunctionRequest));
+      return (runFunctionAction.execute(runBackendStepRequest));
    }
 
 

src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStepTest.java

@@ -0,0 +1,115 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
+
+
+import java.util.Arrays;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for BasicETLCollectSourceFileNamesFunction
+ *******************************************************************************/
+class BasicETLCollectSourceFileNamesStepTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testOneFile() throws Exception
+   {
+      String file   = "/tmp/test1.csv";
+      String result = runTest(file);
+      assertEquals(file, result);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testTwoFiles() throws Exception
+   {
+      String file1  = "/tmp/test1.csv";
+      String file2  = "/tmp/test2.csv";
+      String result = runTest(file1, file2);
+
+      //////////////////////////////////////////////////////////////////////
+      // the names go into a set, so they can come out in either order... //
+      //////////////////////////////////////////////////////////////////////
+      assertTrue(result.equals(file1 + "," + file2) || result.equals((file2 + "," + file1)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testDuplicatedFile() throws Exception
+   {
+      String file   = "/tmp/test1.csv";
+      String result = runTest(file, file);
+      assertEquals(file, result);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String runTest(String... fileNames) throws Exception
+   {
+      QInstance            qInstance           = TestUtils.defineInstance();
+      QBackendStepMetaData backendStepMetaData = new BasicETLCollectSourceFileNamesStep().defineStepMetaData();
+      QProcessMetaData     qProcessMetaData    = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
+      qInstance.addProcess(qProcessMetaData);
+
+      List<QRecord> records = Arrays.stream(fileNames).map(fileName ->
+         new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fileName)).toList();
+
+      RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
+      runBackendStepRequest.setSession(TestUtils.getMockSession());
+      runBackendStepRequest.setStepName(backendStepMetaData.getName());
+      runBackendStepRequest.setProcessName(qProcessMetaData.getName());
+      runBackendStepRequest.setRecords(records);
+
+      RunBackendStepAction runBackendStepAction = new RunBackendStepAction();
+      RunBackendStepResult result               = runBackendStepAction.execute(runBackendStepRequest);
+
+      return ((String) result.getValues().get(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS));
+   }
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java

@@ -25,16 +25,16 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi
 import java.util.List;
 import java.util.stream.Collectors;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
+import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
 import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@@ -81,12 +81,12 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
       QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv");
 
       QProcessMetaData     process = new FilesystemSyncProcess().defineProcessMetaData();
-      QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
+      QBackendStepMetaData step    = process.getBackendStep(FilesystemSyncStep.STEP_NAME);
       qInstance.addProcess(process);
 
-      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
+      step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
-      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
+      step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
-      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
+      step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
 
       ///////////////////////////
       // write some test files //
@@ -99,17 +99,17 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
       printTableListing(archiveBackend, archiveTable);
       printTableListing(processingBackend, processingTable);
 
-      //////////////////////
+      //////////////////
-      // run the function //
+      // run the step //
-      //////////////////////
+      //////////////////
-      RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
+      RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
-      runFunctionRequest.setFunctionName(function.getName());
+      runBackendStepRequest.setStepName(step.getName());
-      runFunctionRequest.setProcessName(process.getName());
+      runBackendStepRequest.setProcessName(process.getName());
-      runFunctionRequest.setSession(TestUtils.getMockSession());
+      runBackendStepRequest.setSession(TestUtils.getMockSession());
 
-      RunFunctionAction runFunctionAction = new RunFunctionAction();
+      RunBackendStepAction runFunctionAction    = new RunBackendStepAction();
-      RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
+      RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
-      System.out.println(runFunctionResult);
+      // System.out.println(runBackendStepResult);
 
       printTableListing(sourceBackend, sourceTable);
       printTableListing(archiveBackend, archiveTable);
@@ -144,12 +144,12 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
       QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv");
 
       QProcessMetaData     process  = new FilesystemSyncProcess().defineProcessMetaData();
-      QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
+      QBackendStepMetaData step = process.getBackendStep(FilesystemSyncStep.STEP_NAME);
       qInstance.addProcess(process);
 
-      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
+      step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
-      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
+      step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
-      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
+      step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
 
       ///////////////////////////
       // write some test files //
@@ -162,17 +162,17 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
       printTableListing(localBackend, archiveTable);
       printTableListing(localBackend, processingTable);
 
-      //////////////////////
+      //////////////////
-      // run the function //
+      // run the step //
-      //////////////////////
+      //////////////////
-      RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
+      RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
-      runFunctionRequest.setFunctionName(function.getName());
+      runBackendStepRequest.setStepName(step.getName());
-      runFunctionRequest.setProcessName(process.getName());
+      runBackendStepRequest.setProcessName(process.getName());
-      runFunctionRequest.setSession(TestUtils.getMockSession());
+      runBackendStepRequest.setSession(TestUtils.getMockSession());
 
-      RunFunctionAction runFunctionAction = new RunFunctionAction();
+      RunBackendStepAction runFunctionAction    = new RunBackendStepAction();
-      RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
+      RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
-      System.out.println(runFunctionResult);
+      // System.out.println(runBackendStepResult);
 
       printTableListing(vendorBackend, sourceTable);
       printTableListing(localBackend, archiveTable);

src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java

@@ -24,14 +24,14 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi
 
 import java.io.File;
 import java.io.IOException;
-import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
+import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
 import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
@@ -58,12 +58,12 @@ class FilesystemSyncProcessTest
       QTableMetaData       archiveTable    = defineTable("archive");
       QTableMetaData       processingTable = defineTable("processing");
       QProcessMetaData     process         = new FilesystemSyncProcess().defineProcessMetaData();
-      QFunctionMetaData function        = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
+      QBackendStepMetaData step            = (QBackendStepMetaData) process.getStep(FilesystemSyncStep.STEP_NAME);
 
-      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
+      step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
-      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
+      step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
-      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
+      step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
-      // function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1);
+      // step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1);
 
       QInstance qInstance = TestUtils.defineInstance();
       qInstance.addTable(sourceTable);
@@ -81,16 +81,16 @@ class FilesystemSyncProcessTest
       writeTestFile(basePath, archiveTable, "2.txt", "x");
 
       //////////////////////
-      // run the function //
+      // run the step //
       //////////////////////
-      RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
+      RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
-      runFunctionRequest.setFunctionName(function.getName());
+      runBackendStepRequest.setStepName(step.getName());
-      runFunctionRequest.setProcessName(process.getName());
+      runBackendStepRequest.setProcessName(process.getName());
-      runFunctionRequest.setSession(TestUtils.getMockSession());
+      runBackendStepRequest.setSession(TestUtils.getMockSession());
 
-      RunFunctionAction runFunctionAction = new RunFunctionAction();
+      RunBackendStepAction runFunctionAction    = new RunBackendStepAction();
-      RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
+      RunBackendStepResult runBackendStepResult = runFunctionAction.execute(runBackendStepRequest);
-      System.out.println(runFunctionResult);
+      // System.out.println(runBackendStepResult);
    }
 
 

src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3ActionTest.java

@@ -0,0 +1,67 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
+
+
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.*;
+
+
+/*******************************************************************************
+ ** Unit test for AbstractS3Action
+ *******************************************************************************/
+class AbstractS3ActionTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testBuildAmazonS3ClientFromBackendMetaData()
+   {
+      String regionName = Regions.AP_SOUTHEAST_3.getName();
+      S3BackendMetaData s3BackendMetaData = new S3BackendMetaData()
+         .withAccessKey("Not a real access key")
+         .withSecretKey("Also not a real key")
+         .withRegion(regionName);
+      AmazonS3 amazonS3 = new AbstractS3Action().buildAmazonS3ClientFromBackendMetaData(s3BackendMetaData);
+      assertEquals(regionName, amazonS3.getRegionName());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testBuildAmazonS3ClientFromBackendMetaDataWrongType()
+   {
+      assertThrows(IllegalArgumentException.class, () ->
+      {
+         new AbstractS3Action().buildAmazonS3ClientFromBackendMetaData(new QBackendMetaData());
+      });
+   }
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java

@@ -52,7 +52,7 @@ class S3BackendMetaDataTest
       System.out.println(JsonUtils.prettyPrint(json));
       System.out.println(json);
       String expectToContain = """
-         {"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","secretKey":null,"accessKey":null,"backendType":"s3","name":"s3","region":null}""";
+         {"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","backendType":"s3","name":"s3"}""";
       assertTrue(json.contains(expectToContain));
    }