Merge branch 'release/0.0.0'

.circleci/config.yml

@@ -1,3 +1,6 @@
+## Deviations from qqq-java library standard circleci config:
+## - To test AWS S3, uses localstsack executor, orb, and startup step
+## - This docker image doesn't have java-17, so we install (and cache) jvm-17
 version: 2.1
 
 executors:
@@ -8,8 +11,18 @@ executors:
 
 orbs:
   slack: circleci/slack@4.10.1
+  localstack: localstack/platform@1.0
 
 commands:
+  install_java17:
+    steps:
+      - run:
+          name: Install Java 17
+          command: |
+            sudo add-apt-repository -y ppa:openjdk-r/ppa
+            sudo apt install -y openjdk-17-jdk
+            sudo rm /etc/alternatives/java
+            sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java
   run_maven:
     parameters:
       maven_subcommand:
@@ -39,16 +52,20 @@ commands:
 
 jobs:
   mvn_test:
-    executor: java17
+    executor: localstack/default
     steps:
+      - localstack/startup
+      - install_java17
       - run_maven:
           maven_subcommand: test
       - slack/notify:
           event: fail
 
   mvn_deploy:
-    executor: java17
+    executor: localstack/default
     steps:
+      - localstack/startup
+      - install_java17
       - run_maven:
           maven_subcommand: deploy
       - slack/notify:

pom.xml

@@ -25,7 +25,7 @@
 
    <groupId>com.kingsrook.qqq</groupId>
    <artifactId>qqq-backend-module-filesystem</artifactId>
-   <version>0.0.0-SNAPSHOT</version>
+   <version>0.0.0</version>
 
    <scm>
      <connection>scm:git:git@github.com:Kingsrook/qqq-backend-module-rdbms.git</connection>
@@ -51,7 +51,7 @@
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
          <artifactId>qqq-backend-core</artifactId>
-         <version>0.0.0-SNAPSHOT</version>
+         <version>0.0.0</version>
       </dependency>
 
       <!-- 3rd party deps specifically for this module -->
@@ -61,21 +61,9 @@
          <version>1.12.243</version>
       </dependency>
       <dependency>
-         <groupId>org.testcontainers</groupId>
+         <groupId>cloud.localstack</groupId>
-         <artifactId>testcontainers</artifactId>
+         <artifactId>localstack-utils</artifactId>
-         <version>1.17.2</version>
+         <version>0.2.20</version>
-         <scope>test</scope>
-      </dependency>
-      <dependency>
-         <groupId>org.testcontainers</groupId>
-         <artifactId>localstack</artifactId>
-         <version>1.17.2</version>
-         <scope>test</scope>
-      </dependency>
-      <dependency>
-         <groupId>org.testcontainers</groupId>
-         <artifactId>junit-jupiter</artifactId>
-         <version>1.17.2</version>
          <scope>test</scope>
       </dependency>
 

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java

@@ -0,0 +1,39 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.base;
+
+
+import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
+
+
+/*******************************************************************************
+ ** Interface to add additional functionality commonly among the various filesystem
+ ** module implementations.
+ *******************************************************************************/
+public interface FilesystemBackendModuleInterface<FILE>
+{
+   /*******************************************************************************
+    ** For filesystem backends, get the module-specific action base-class, that helps
+    ** with functions like listing and deleting files.
+    *******************************************************************************/
+   AbstractBaseFilesystemAction<FILE> getActionBase();
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemRecordBackendDetailFields.java

@@ -0,0 +1,32 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.base;
+
+
+/*******************************************************************************
+ ** Define the field names (keys) to be used in the backendDetails structure
+ ** under Records from this (these) modules.
+ *******************************************************************************/
+public interface FilesystemRecordBackendDetailFields
+{
+   String FULL_PATH = "fullPath";
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java

@@ -0,0 +1,294 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.base.actions;
+
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
+import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
+import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
+import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+
+/*******************************************************************************
+ ** Base class for all Filesystem actions across all modules.
+ **
+ ** @param FILE The class that represents a file in the sub-module.  e.g.,
+ *               a java.io.File, or an S3Object.
+ *******************************************************************************/
+public abstract class AbstractBaseFilesystemAction<FILE>
+{
+   private static final Logger LOG = LogManager.getLogger(AbstractBaseFilesystemAction.class);
+
+
+
+   /*******************************************************************************
+    ** List the files for a table - to be implemented in module-specific subclasses.
+    *******************************************************************************/
+   public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase);
+
+   /*******************************************************************************
+    ** Read the contents of a file - to be implemented in module-specific subclasses.
+    *******************************************************************************/
+   public abstract InputStream readFile(FILE file) throws IOException;
+
+   /*******************************************************************************
+    ** Write a file - to be implemented in module-specific subclasses.
+    *******************************************************************************/
+   public abstract void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException;
+
+   /*******************************************************************************
+    ** Get a string that represents the full path to a file.
+    *******************************************************************************/
+   public abstract String getFullPathForFile(FILE file);
+
+   /*******************************************************************************
+    ** In contrast with the DeleteAction, which deletes RECORDS - this is a
+    ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
+    ** e.g., for post-ETL.
+    **
+    ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
+    *******************************************************************************/
+   public abstract void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException;
+
+   /*******************************************************************************
+    ** Move a file from a source path, to a destination path.
+    **
+    ** @throws FilesystemException if the move is known to have failed
+    *******************************************************************************/
+   public abstract void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException;
+
+   /*******************************************************************************
+    ** e.g., with a base path of /foo/
+    ** and a table path of /bar/
+    ** and a file at /foo/bar/baz.txt
+    ** give us just the baz.txt part.
+    *******************************************************************************/
+   public abstract String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData sourceBackend, QTableMetaData sourceTable);
+
+
+
+   /*******************************************************************************
+    ** Append together the backend's base path (if present), with a table's base
+    ** path (again, if present).
+    *******************************************************************************/
+   public String getFullBasePath(QTableMetaData table, QBackendMetaData backendBase)
+   {
+      AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase);
+      String                            fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : "";
+
+      AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
+      if(StringUtils.hasContent(tableDetails.getBasePath()))
+      {
+         fullPath += File.separatorChar + tableDetails.getBasePath();
+      }
+
+      fullPath += File.separatorChar;
+      fullPath = stripDuplicatedSlashes(fullPath);
+
+      return fullPath;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String stripDuplicatedSlashes(String path)
+   {
+      if(path == null)
+      {
+         return (null);
+      }
+
+      return (path.replaceAll("//+", "/"));
+   }
+
+
+
+   /*******************************************************************************
+    ** Get the backend metaData, type-checked as the requested type.
+    *******************************************************************************/
+   protected <T extends AbstractFilesystemBackendMetaData> T getBackendMetaData(Class<T> outputClass, QBackendMetaData metaData)
+   {
+      if(!(outputClass.isInstance(metaData)))
+      {
+         throw new IllegalArgumentException("MetaData was not of expected type (was " + metaData.getClass().getSimpleName() + ")");
+      }
+      return outputClass.cast(metaData);
+   }
+
+
+
+   /*******************************************************************************
+    ** Get the backendDetails out of a table, type-checked as the requested type
+    *******************************************************************************/
+   protected <T extends AbstractFilesystemTableBackendDetails> T getTableBackendDetails(Class<T> outputClass, QTableMetaData tableMetaData)
+   {
+      QTableBackendDetails tableBackendDetails = tableMetaData.getBackendDetails();
+      if(!(outputClass.isInstance(tableBackendDetails)))
+      {
+         throw new IllegalArgumentException("Table backend details was not of expected type (was " + tableBackendDetails.getClass().getSimpleName() + ")");
+      }
+      return outputClass.cast(tableBackendDetails);
+   }
+
+
+
+   /*******************************************************************************
+    ** Generic implementation of the execute method from the QueryInterface
+    *******************************************************************************/
+   public QueryResult executeQuery(QueryRequest queryRequest) throws QException
+   {
+      preAction(queryRequest.getBackend());
+
+      try
+      {
+         QueryResult   rs      = new QueryResult();
+         List<QRecord> records = new ArrayList<>();
+         rs.setRecords(records);
+
+         QTableMetaData                        table        = queryRequest.getTable();
+         AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
+         List<FILE>                            files        = listFiles(table, queryRequest.getBackend());
+
+         for(FILE file : files)
+         {
+            LOG.info("Processing file: " + getFullPathForFile(file));
+            switch(tableDetails.getRecordFormat())
+            {
+               case CSV:
+               {
+                  String fileContents = IOUtils.toString(readFile(file));
+                  fileContents = customizeFileContentsAfterReading(table, fileContents);
+
+                  List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
+                  addBackendDetailsToRecords(recordsInFile, file);
+
+                  records.addAll(recordsInFile);
+                  break;
+               }
+               case JSON:
+               {
+                  String fileContents = IOUtils.toString(readFile(file));
+                  fileContents = customizeFileContentsAfterReading(table, fileContents);
+
+                  List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
+                  addBackendDetailsToRecords(recordsInFile, file);
+
+                  records.addAll(recordsInFile);
+                  break;
+               }
+               default:
+               {
+                  throw new NotImplementedException("Filesystem record format " + tableDetails.getRecordFormat() + " is not yet implemented");
+               }
+            }
+         }
+
+         return rs;
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error executing query", e);
+         throw new QException("Error executing query", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Add backend details to records about the file that they are in.
+    *******************************************************************************/
+   protected void addBackendDetailsToRecords(List<QRecord> recordsInFile, FILE file)
+   {
+      recordsInFile.forEach(record ->
+      {
+         record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, getFullPathForFile(file));
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** Method that subclasses can override to add pre-action things (e.g., setting up
+    ** s3 client).
+    *******************************************************************************/
+   public void preAction(QBackendMetaData backendMetaData)
+   {
+      /////////////////////////////////////////////////////////////////////
+      // noop in base class - subclasses can add functionality if needed //
+      /////////////////////////////////////////////////////////////////////
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String customizeFileContentsAfterReading(QTableMetaData table, String fileContents) throws QException
+   {
+      Optional<QCodeReference> optionalCustomizer = table.getCustomizer("postFileRead");
+      if(optionalCustomizer.isEmpty())
+      {
+         return (fileContents);
+      }
+      QCodeReference customizer = optionalCustomizer.get();
+
+      try
+      {
+         Class<?> customizerClass = Class.forName(customizer.getName());
+
+         @SuppressWarnings("unchecked")
+         Function<String, String> function = (Function<String, String>) customizerClass.getConstructor().newInstance();
+
+         return function.apply(fileContents);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error customizing file contents", e));
+      }
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java

@@ -0,0 +1,80 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+
+
+/*******************************************************************************
+ ** Base class for all BackendMetaData for all filesystem-style backend modules.
+ *******************************************************************************/
+public class AbstractFilesystemBackendMetaData extends QBackendMetaData
+{
+   private String basePath;
+
+
+
+   /*******************************************************************************
+    ** Default Constructor.
+    *******************************************************************************/
+   public AbstractFilesystemBackendMetaData()
+   {
+      super();
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for basePath
+    **
+    *******************************************************************************/
+   public String getBasePath()
+   {
+      return (basePath);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for basePath
+    **
+    *******************************************************************************/
+   public void setBasePath(String basePath)
+   {
+      this.basePath = basePath;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for basePath
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public <T extends AbstractFilesystemBackendMetaData> T withBasePath(String basePath)
+   {
+      this.basePath = basePath;
+      return (T) this;
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java

@@ -0,0 +1,178 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
+
+
+/*******************************************************************************
+ ** Extension of QTableBackendDetails, with details specific to a Filesystem table.
+ *******************************************************************************/
+public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
+{
+   private String       basePath;
+   private String       glob;
+   private RecordFormat recordFormat;
+   private Cardinality  cardinality;
+
+
+
+   /*******************************************************************************
+    ** Getter for basePath
+    **
+    *******************************************************************************/
+   public String getBasePath()
+   {
+      return basePath;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for basePath
+    **
+    *******************************************************************************/
+   public void setBasePath(String basePath)
+   {
+      this.basePath = basePath;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent Setter for basePath
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public <T extends AbstractFilesystemTableBackendDetails> T withBasePath(String basePath)
+   {
+      this.basePath = basePath;
+      return (T) this;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for glob
+    **
+    *******************************************************************************/
+   public String getGlob()
+   {
+      return glob;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for glob
+    **
+    *******************************************************************************/
+   public void setGlob(String glob)
+   {
+      this.glob = glob;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent Setter for glob
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public <T extends AbstractFilesystemTableBackendDetails> T withGlob(String glob)
+   {
+      this.glob = glob;
+      return (T) this;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for recordFormat
+    **
+    *******************************************************************************/
+   public RecordFormat getRecordFormat()
+   {
+      return recordFormat;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for recordFormat
+    **
+    *******************************************************************************/
+   public void setRecordFormat(RecordFormat recordFormat)
+   {
+      this.recordFormat = recordFormat;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent Setter for recordFormat
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public <T extends AbstractFilesystemTableBackendDetails> T withRecordFormat(RecordFormat recordFormat)
+   {
+      this.recordFormat = recordFormat;
+      return ((T) this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for cardinality
+    **
+    *******************************************************************************/
+   public Cardinality getCardinality()
+   {
+      return cardinality;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for cardinality
+    **
+    *******************************************************************************/
+   public void setCardinality(Cardinality cardinality)
+   {
+      this.cardinality = cardinality;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent Setter for cardinality
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public <T extends AbstractFilesystemTableBackendDetails> T withCardinality(Cardinality cardinality)
+   {
+      this.cardinality = cardinality;
+      return ((T) this);
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/Cardinality.java

@@ -0,0 +1,33 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
+
+
+/*******************************************************************************
+ ** Mode for filesystem backends:  are all records in a single file, or are there
+ ** many files?
+ *******************************************************************************/
+public enum Cardinality
+{
+   ONE,
+   MANY
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/RecordFormat.java

@@ -0,0 +1,34 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
+
+
+/*******************************************************************************
+ ** How are records stored in the files in a filesystem backend?  CSV, JSON,
+ ** future types may XML, or more exotic ones, like "BINARY" or "TEXT" (e.g., 1
+ ** record and 1 field per-file)
+ *******************************************************************************/
+public enum RecordFormat
+{
+   CSV,
+   JSON
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/exceptions/FilesystemException.java

@@ -0,0 +1,52 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.exceptions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+
+
+/*******************************************************************************
+ ** Checked exception to be thrown from actions within this module.
+ *******************************************************************************/
+public class FilesystemException extends QException
+{
+   /*******************************************************************************
+    ** Constructor of message
+    **
+    *******************************************************************************/
+   public FilesystemException(String message)
+   {
+      super(message);
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor of message & cause
+    **
+    *******************************************************************************/
+   public FilesystemException(String message, Throwable cause)
+   {
+      super(message, cause);
+   }
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java

@@ -0,0 +1,142 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.local;
+
+
+import java.io.File;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
+import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
+import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
+import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
+import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
+import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
+import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
+import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
+import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction;
+import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction;
+import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction;
+import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemUpdateAction;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+
+/*******************************************************************************
+ ** QQQ Backend module for working with (local) Filesystems.
+ *******************************************************************************/
+public class FilesystemBackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface
+{
+   private static final Logger LOG = LogManager.getLogger(FilesystemBackendModule.class);
+
+
+
+   /*******************************************************************************
+    ** For filesystem backends, get the module-specific action base-class, that helps
+    ** with functions like listing and deleting files.
+    *******************************************************************************/
+   @Override
+   public AbstractBaseFilesystemAction<File> getActionBase()
+   {
+      return (new AbstractFilesystemAction());
+   }
+
+
+
+   /*******************************************************************************
+    ** Method where a backend module must be able to provide its type (name).
+    *******************************************************************************/
+   @Override
+   public String getBackendType()
+   {
+      return ("filesystem");
+   }
+
+
+
+   /*******************************************************************************
+    ** Method to identify the class used for backend meta data for this module.
+    *******************************************************************************/
+   @Override
+   public Class<? extends QBackendMetaData> getBackendMetaDataClass()
+   {
+      return (FilesystemBackendMetaData.class);
+   }
+
+
+
+   /*******************************************************************************
+    ** Method to identify the class used for table-backend details for this module.
+    *******************************************************************************/
+   @Override
+   public Class<? extends QTableBackendDetails> getTableBackendDetailsClass()
+   {
+      return FilesystemTableBackendDetails.class;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QueryInterface getQueryInterface()
+   {
+      return new FilesystemQueryAction();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public InsertInterface getInsertInterface()
+   {
+      return (new FilesystemInsertAction());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public UpdateInterface getUpdateInterface()
+   {
+      return (new FilesystemUpdateAction());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public DeleteInterface getDeleteInterface()
+   {
+      return (new FilesystemDeleteAction());
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java

@@ -0,0 +1,186 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.local.actions;
+
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
+import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
+import org.apache.commons.io.FileUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+
+/*******************************************************************************
+ ** Base class for all (local) Filesystem actions
+ *******************************************************************************/
+public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
+{
+   private static final Logger LOG = LogManager.getLogger(AbstractFilesystemAction.class);
+
+
+
+   /*******************************************************************************
+    ** List the files for this table.
+    *******************************************************************************/
+   @Override
+   public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase)
+   {
+      // todo - needs rewritten to do globbing...
+      String fullPath  = getFullBasePath(table, backendBase);
+      File   directory = new File(fullPath);
+      File[] files     = directory.listFiles();
+
+      if(files == null)
+      {
+         return Collections.emptyList();
+      }
+
+      return (Arrays.stream(files).filter(File::isFile).toList());
+   }
+
+
+
+   /*******************************************************************************
+    ** Read the contents of a file.
+    *******************************************************************************/
+   @Override
+   public InputStream readFile(File file) throws IOException
+   {
+      return (new FileInputStream(file));
+   }
+
+
+
+   /*******************************************************************************
+    ** Write a file - to be implemented in module-specific subclasses.
+    *******************************************************************************/
+   @Override
+   public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
+   {
+      FileUtils.writeByteArrayToFile(new File(path), contents);
+   }
+
+
+
+   /*******************************************************************************
+    ** Get a string that represents the full path to a file.
+    *******************************************************************************/
+   @Override
+   public String getFullPathForFile(File file)
+   {
+      return (file.getAbsolutePath());
+   }
+
+
+
+   /*******************************************************************************
+    ** In contrast with the DeleteAction, which deletes RECORDS - this is a
+    ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
+    ** e.g., for post-ETL.
+    **
+    ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
+    *******************************************************************************/
+   @Override
+   public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
+   {
+      File file = new File(fileReference);
+      if(!file.exists())
+      {
+         //////////////////////////////////////////////////////////////////////////////////////////////
+         // if the file doesn't exist, just exit with noop.  don't throw an error - that should only //
+         // happen if the "contract" of the method is broken, and the file still exists              //
+         //////////////////////////////////////////////////////////////////////////////////////////////
+         LOG.debug("Not deleting file [{}], because it does not exist.", file);
+         return;
+      }
+
+      if(!file.delete())
+      {
+         throw (new FilesystemException("Failed to delete file: " + fileReference));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Move a file from a source path, to a destination path.
+    **
+    ** @param destination assumed to be a file path - not a directory
+    ** @throws FilesystemException if the delete is known to have failed
+    *******************************************************************************/
+   @Override
+   public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException
+   {
+      File sourceFile        = new File(source);
+      File destinationFile   = new File(destination);
+      File destinationParent = destinationFile.getParentFile();
+
+      if(!sourceFile.exists())
+      {
+         throw (new FilesystemException("Cannot move file " + source + ", as it does not exist."));
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////
+      // if the destination folder doesn't exist, try to make it - and fail if that fails //
+      //////////////////////////////////////////////////////////////////////////////////////
+      if(!destinationParent.exists())
+      {
+         LOG.debug("Making destination directory {} for move", destinationParent.getAbsolutePath());
+         if(!destinationParent.mkdirs())
+         {
+            throw (new FilesystemException("Failed to make destination directory " + destinationParent.getAbsolutePath() + " to move " + source + " into."));
+         }
+      }
+
+      if(!sourceFile.renameTo(destinationFile))
+      {
+         throw (new FilesystemException("Failed to move (rename) file " + source + " to " + destination));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** e.g., with a base path of /foo/
+    ** and a table path of /bar/
+    ** and a file at /foo/bar/baz.txt
+    ** give us just the baz.txt part.
+    *******************************************************************************/
+   @Override
+   public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table)
+   {
+      String tablePath    = getFullBasePath(table, backend);
+      String strippedPath = filePath.replaceFirst(".*" + tablePath, "");
+      return (strippedPath);
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java

@@ -0,0 +1,60 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.local.actions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
+import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
+import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
+import org.apache.commons.lang.NotImplementedException;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class FilesystemDeleteAction implements DeleteInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public DeleteResult execute(DeleteRequest deleteRequest) throws QException
+   {
+      throw new NotImplementedException("Filesystem delete not implemented");
+      /*
+      try
+      {
+         DeleteResult rs = new DeleteResult();
+         QTableMetaData table = deleteRequest.getTable();
+
+
+         // return rs;
+      }
+      catch(Exception e)
+      {
+         throw new QException("Error executing delete: " + e.getMessage(), e);
+      }
+      */
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java

@@ -0,0 +1,62 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.local.actions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
+import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
+import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
+import org.apache.commons.lang.NotImplementedException;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class FilesystemInsertAction implements InsertInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public InsertResult execute(InsertRequest insertRequest) throws QException
+   {
+      throw new NotImplementedException("Filesystem insert not implemented");
+      /*
+      try
+      {
+         InsertResult rs = new InsertResult();
+         QTableMetaData table = insertRequest.getTable();
+
+         List<QRecord> recordsWithStatus = new ArrayList<>();
+         rs.setRecords(recordsWithStatus);
+
+         // return rs;
+      }
+      catch(Exception e)
+      {
+         throw new QException("Error executing insert: " + e.getMessage(), e);
+      }
+      */
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryAction.java

@@ -0,0 +1,45 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.local.actions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
+import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
+import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class FilesystemQueryAction extends AbstractFilesystemAction implements QueryInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QueryResult execute(QueryRequest queryRequest) throws QException
+   {
+      return executeQuery(queryRequest);
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java

@@ -0,0 +1,63 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.local.actions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
+import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
+import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
+import org.apache.commons.lang.NotImplementedException;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class FilesystemUpdateAction implements UpdateInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public UpdateResult execute(UpdateRequest updateRequest) throws QException
+   {
+      throw new NotImplementedException("Filesystem update not implemented");
+      /*
+      try
+      {
+         UpdateResult rs = new UpdateResult();
+         QTableMetaData table = updateRequest.getTable();
+
+         List<QRecord> records = new ArrayList<>();
+         rs.setRecords(records);
+
+
+         // return rs;
+      }
+      catch(Exception e)
+      {
+         throw new QException("Error executing update: " + e.getMessage(), e);
+      }
+      */
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java

@@ -0,0 +1,45 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata;
+
+
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule;
+
+
+/*******************************************************************************
+ ** (local) Filesystem backend meta data.
+ *******************************************************************************/
+public class FilesystemBackendMetaData extends AbstractFilesystemBackendMetaData
+{
+
+
+   /*******************************************************************************
+    ** Default Constructor.
+    *******************************************************************************/
+   public FilesystemBackendMetaData()
+   {
+      super();
+      setBackendType(FilesystemBackendModule.class);
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemTableBackendDetails.java

@@ -0,0 +1,44 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata;
+
+
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
+import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule;
+
+
+/*******************************************************************************
+ ** (local) Filesystem specific Extension of QTableBackendDetails
+ *******************************************************************************/
+public class FilesystemTableBackendDetails extends AbstractFilesystemTableBackendDetails
+{
+
+   /*******************************************************************************
+    ** Default Constructor.
+    *******************************************************************************/
+   public FilesystemTableBackendDetails()
+   {
+      super();
+      setBackendType(FilesystemBackendModule.class);
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java

@@ -0,0 +1,135 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
+
+
+import java.io.File;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
+import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
+import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+
+/*******************************************************************************
+ ** Function body for performing the Cleanup step of a basic ETL process - e.g.,
+ ** after the loading, delete or move the processed file(s).
+ *******************************************************************************/
+public class BasicETLCleanupSourceFilesFunction implements FunctionBody
+{
+   private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesFunction.class);
+
+   public static final String FIELD_MOVE_OR_DELETE        = "moveOrDelete";
+   public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves";
+
+   public static final String VALUE_MOVE    = "move";
+   public static final String VALUE_DELETE  = "delete";
+   public static final String FUNCTION_NAME = "cleanupSourceFiles";
+
+
+
+   /*******************************************************************************
+    ** Execute the function - using the request as input, and the result as output.
+    *******************************************************************************/
+   @Override
+   public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
+   {
+      String                  sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
+      QTableMetaData          table           = runFunctionRequest.getInstance().getTable(sourceTableName);
+      QBackendMetaData        backend         = runFunctionRequest.getInstance().getBackendForTable(sourceTableName);
+      QBackendModuleInterface module          = new QBackendModuleDispatcher().getQBackendModule(backend);
+
+      if(!(module instanceof FilesystemBackendModuleInterface filesystemModule))
+      {
+         throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function."));
+      }
+      AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase();
+      actionBase.preAction(backend);
+
+      String sourceFilePaths = runFunctionRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS);
+      if(!StringUtils.hasContent(sourceFilePaths))
+      {
+         LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS + "]");
+         return;
+      }
+
+      String[] sourceFiles = sourceFilePaths.split(",");
+      for(String sourceFile : sourceFiles)
+      {
+         String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE);
+         if(VALUE_DELETE.equals(moveOrDelete))
+         {
+            actionBase.deleteFile(runFunctionRequest.getInstance(), table, sourceFile);
+         }
+         else if(VALUE_MOVE.equals(moveOrDelete))
+         {
+            String destinationForMoves = runFunctionRequest.getValueString(FIELD_DESTINATION_FOR_MOVES);
+            if(!StringUtils.hasContent(destinationForMoves))
+            {
+               throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value."));
+            }
+            String filePathWithoutBase = actionBase.stripBackendAndTableBasePathsFromFileName(sourceFile, backend, table);
+            String destinationPath     = destinationForMoves + File.separator + filePathWithoutBase;
+            actionBase.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath);
+         }
+         else
+         {
+            throw (new QException("Unexpected value [" + moveOrDelete + "] for field [" + FIELD_MOVE_OR_DELETE + "].  "
+               + "Must be either [" + VALUE_MOVE + "] or [" + VALUE_DELETE + "]."));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** define the metaData that describes this function
+    *******************************************************************************/
+   public QFunctionMetaData defineFunctionMetaData()
+   {
+      return (new QFunctionMetaData()
+         .withName(FUNCTION_NAME)
+         .withCode(new QCodeReference()
+            .withName(this.getClass().getName())
+            .withCodeType(QCodeType.JAVA)
+            .withCodeUsage(QCodeUsage.FUNCTION))
+         .withInputData(new QFunctionInputMetaData()
+            .addField(new QFieldMetaData("moveOrDelete", QFieldType.STRING))
+            .addField(new QFieldMetaData("destinationForMoves", QFieldType.STRING))));
+   }
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java

@@ -0,0 +1,84 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
+
+
+import java.util.Set;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
+
+
+/*******************************************************************************
+ ** Function body for collecting the file names that were discovered in the
+ ** Extract step.  These will be lost during the transform, so we capture them here,
+ ** so that our Clean function can move or delete them.
+ **
+ ** TODO - need unit test!!
+ *******************************************************************************/
+public class BasicETLCollectSourceFileNamesFunction implements FunctionBody
+{
+   public static final String FUNCTION_NAME           = "collectSourceFileNames";
+   public static final String FIELD_SOURCE_FILE_PATHS = "sourceFilePaths";
+
+
+
+   /*******************************************************************************
+    ** Execute the function - using the request as input, and the result as output.
+    *******************************************************************************/
+   @Override
+   public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
+   {
+      Set<String> sourceFiles = runFunctionRequest.getRecords().stream()
+         .map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))
+         .collect(Collectors.toSet());
+      runFunctionResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles));
+   }
+
+
+
+   /*******************************************************************************
+    ** define the metaData that describes this function
+    *******************************************************************************/
+   public QFunctionMetaData defineFunctionMetaData()
+   {
+      return (new QFunctionMetaData()
+         .withName(FUNCTION_NAME)
+         .withCode(new QCodeReference()
+            .withName(this.getClass().getName())
+            .withCodeType(QCodeType.JAVA)
+            .withCodeUsage(QCodeUsage.FUNCTION))
+         .withOutputMetaData(new QFunctionOutputMetaData()
+            .addField(new QFieldMetaData(FIELD_SOURCE_FILE_PATHS, QFieldType.STRING))));
+   }
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java

@@ -0,0 +1,138 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync;
+
+
+import java.io.File;
+import java.io.InputStream;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
+import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+
+/*******************************************************************************
+ ** Function body for collecting the file names that were discovered in the
+ ** Extract step.  These will be lost during the transform, so we capture them here,
+ ** so that our Clean function can move or delete them.
+ **
+ *******************************************************************************/
+public class FilesystemSyncFunction implements FunctionBody
+{
+   private static final Logger LOG = LogManager.getLogger(FilesystemSyncFunction.class);
+
+   public static final String FUNCTION_NAME = "sync";
+
+
+
+   /*******************************************************************************
+    ** Execute the function - using the request as input, and the result as output.
+    *******************************************************************************/
+   @Override
+   public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
+   {
+      QTableMetaData sourceTable     = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
+      QTableMetaData archiveTable    = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
+      QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
+
+      QBackendMetaData                 sourceBackend    = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName());
+      FilesystemBackendModuleInterface sourceModule     = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
+      AbstractBaseFilesystemAction     sourceActionBase = sourceModule.getActionBase();
+      sourceActionBase.preAction(sourceBackend);
+      Map<String, Object> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
+
+      QBackendMetaData                 archiveBackend    = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName());
+      FilesystemBackendModuleInterface archiveModule     = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
+      AbstractBaseFilesystemAction     archiveActionBase = archiveModule.getActionBase();
+      archiveActionBase.preAction(archiveBackend);
+      Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet();
+
+      QBackendMetaData                 processingBackend    = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName());
+      FilesystemBackendModuleInterface processingModule     = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
+      AbstractBaseFilesystemAction     processingActionBase = processingModule.getActionBase();
+      processingActionBase.preAction(processingBackend);
+
+      Integer maxFilesToSync = runFunctionRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
+      int syncedFileCount = 0;
+      for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet())
+      {
+         try
+         {
+            String sourceFileName = sourceEntry.getKey();
+            if(!archiveFiles.contains(sourceFileName))
+            {
+               LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]");
+               InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue());
+               byte[]      bytes       = inputStream.readAllBytes();
+
+               String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
+               archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
+
+               String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
+               processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
+               syncedFileCount++;
+
+               if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)
+               {
+                  LOG.info("Breaking after syncing " + syncedFileCount + " files");
+                  break;
+               }
+            }
+         }
+         catch(Exception e)
+         {
+            LOG.error("Error processing file: " + sourceEntry, e);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Map<String, Object> getFileNames(AbstractBaseFilesystemAction actionBase, QTableMetaData table, QBackendMetaData backend)
+   {
+      List<Object>        files = actionBase.listFiles(table, backend);
+      Map<String, Object> rs    = new LinkedHashMap<>();
+
+      for(Object file : files)
+      {
+         String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table);
+         rs.put(fileName, file);
+      }
+
+      return (rs);
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java

@@ -0,0 +1,83 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+
+
+/*******************************************************************************
+ ** Definition for Filesystem sync process.
+ **
+ ** Job is to:
+ ** - list all files in the source table.
+ ** - list all files in the archive table.
+ ** - if any files exist in the source, but not in the archive, then:
+ **   - copy the file to both the archive and the processing table.
+ **
+ ** The maxFilesToArchive field can be used to only sync up to that many files
+ ** (an help an initial sync, if you want to do it in smaller batches)
+ **
+ ** The idea being, that the source is read-only, and we want to move files out of
+ ** processing after they've been processed - and the archive is what we can have
+ ** in-between the two.
+ *******************************************************************************/
+public class FilesystemSyncProcess
+{
+   public static final String PROCESS_NAME = "filesystem.sync";
+
+   public static final String FIELD_SOURCE_TABLE         = "sourceTable";
+   public static final String FIELD_ARCHIVE_TABLE        = "archiveTable";
+   public static final String FIELD_PROCESSING_TABLE     = "processingTable";
+   public static final String FIELD_MAX_FILES_TO_ARCHIVE = "maxFilesToArchive";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QProcessMetaData defineProcessMetaData()
+   {
+      QFunctionMetaData syncFunction = new QFunctionMetaData()
+         .withName(FilesystemSyncFunction.FUNCTION_NAME)
+         .withCode(new QCodeReference()
+            .withName(FilesystemSyncFunction.class.getName())
+            .withCodeType(QCodeType.JAVA)
+            .withCodeUsage(QCodeUsage.FUNCTION))
+         .withInputData(new QFunctionInputMetaData()
+            .addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING))
+            .addField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING))
+            .addField(new QFieldMetaData(FIELD_MAX_FILES_TO_ARCHIVE, QFieldType.INTEGER).withDefaultValue(Integer.MAX_VALUE))
+            .addField(new QFieldMetaData(FIELD_PROCESSING_TABLE, QFieldType.STRING)));
+
+      return new QProcessMetaData()
+         .withName(PROCESS_NAME)
+         .addFunction(syncFunction);
+   }
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java

@@ -0,0 +1,138 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3;
+
+
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
+import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
+import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
+import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
+import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
+import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
+import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
+import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
+import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction;
+import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction;
+import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction;
+import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction;
+import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
+
+
+/*******************************************************************************
+ ** QQQ Backend module for working with AWS S3 filesystems
+ *******************************************************************************/
+public class S3BackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface
+{
+
+
+   /*******************************************************************************
+    ** For filesystem backends, get the module-specific action base-class, that helps
+    ** with functions like listing and deleting files.
+    *******************************************************************************/
+   @Override
+   public AbstractBaseFilesystemAction<S3ObjectSummary> getActionBase()
+   {
+      return (new AbstractS3Action());
+   }
+
+
+
+   /*******************************************************************************
+    ** Method where a backend module must be able to provide its type (name).
+    *******************************************************************************/
+   @Override
+   public String getBackendType()
+   {
+      return ("s3");
+   }
+
+
+
+   /*******************************************************************************
+    ** Method to identify the class used for backend meta data for this module.
+    *******************************************************************************/
+   @Override
+   public Class<? extends QBackendMetaData> getBackendMetaDataClass()
+   {
+      return (S3BackendMetaData.class);
+   }
+
+
+
+   /*******************************************************************************
+    ** Method to identify the class used for table-backend details for this module.
+    *******************************************************************************/
+   @Override
+   public Class<? extends QTableBackendDetails> getTableBackendDetailsClass()
+   {
+      return (S3TableBackendDetails.class);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QueryInterface getQueryInterface()
+   {
+      return new S3QueryAction();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public InsertInterface getInsertInterface()
+   {
+      return (new S3InsertAction());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public UpdateInterface getUpdateInterface()
+   {
+      return (new S3UpdateAction());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public DeleteInterface getDeleteInterface()
+   {
+      return (new S3DeleteAction());
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java

@@ -0,0 +1,237 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
+
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
+import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
+import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+
+/*******************************************************************************
+ ** Base class for all S3 filesystem actions
+ *******************************************************************************/
+public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSummary>
+{
+   private static final Logger LOG = LogManager.getLogger(AbstractS3Action.class);
+
+   private S3Utils s3Utils;
+
+
+
+   /*******************************************************************************
+    ** Setup the s3 utils object to be used for this action.
+    *******************************************************************************/
+   @Override
+   public void preAction(QBackendMetaData backendMetaData)
+   {
+      super.preAction(backendMetaData);
+
+      if(s3Utils != null)
+      {
+         LOG.debug("s3Utils object is already set - not re-setting it.");
+         return;
+      }
+
+      S3BackendMetaData     s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendMetaData);
+      AmazonS3ClientBuilder clientBuilder     = AmazonS3ClientBuilder.standard();
+
+      if(StringUtils.hasContent(s3BackendMetaData.getAccessKey()) && StringUtils.hasContent(s3BackendMetaData.getSecretKey()))
+      {
+         BasicAWSCredentials credentials = new BasicAWSCredentials(s3BackendMetaData.getAccessKey(), s3BackendMetaData.getSecretKey());
+         clientBuilder.setCredentials(new AWSStaticCredentialsProvider(credentials));
+      }
+
+      if(StringUtils.hasContent(s3BackendMetaData.getRegion()))
+      {
+         clientBuilder.setRegion(s3BackendMetaData.getRegion());
+      }
+
+      s3Utils = new S3Utils();
+      s3Utils.setAmazonS3(clientBuilder.build());
+   }
+
+
+
+   /*******************************************************************************
+    ** Set the S3Utils object.
+    *******************************************************************************/
+   public void setS3Utils(S3Utils s3Utils)
+   {
+      this.s3Utils = s3Utils;
+   }
+
+
+
+   /*******************************************************************************
+    ** Internal accessor for the s3Utils object - should always use this, not the field.
+    *******************************************************************************/
+   private S3Utils getS3Utils()
+   {
+      if(s3Utils == null)
+      {
+         s3Utils = new S3Utils();
+      }
+
+      return s3Utils;
+   }
+
+
+
+   /*******************************************************************************
+    ** List the files for a table.
+    *******************************************************************************/
+   @Override
+   public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase)
+   {
+      S3BackendMetaData                     s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase);
+      AbstractFilesystemTableBackendDetails tableDetails      = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
+
+      String fullPath   = getFullBasePath(table, backendBase);
+      String bucketName = s3BackendMetaData.getBucketName();
+      String glob       = tableDetails.getGlob();
+
+      ////////////////////////////////////////////////////////////////////
+      // todo - look at metadata to configure the s3 client here?       //
+      ////////////////////////////////////////////////////////////////////
+      return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob);
+   }
+
+
+
+   /*******************************************************************************
+    ** Read the contents of a file.
+    *******************************************************************************/
+   @Override
+   public InputStream readFile(S3ObjectSummary s3ObjectSummary) throws IOException
+   {
+      return (getS3Utils().getObjectAsInputStream(s3ObjectSummary));
+   }
+
+
+
+   /*******************************************************************************
+    ** Write a file - to be implemented in module-specific subclasses.
+    *******************************************************************************/
+   @Override
+   public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException
+   {
+      path = stripLeadingSlash(stripDuplicatedSlashes(path));
+      String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName();
+      getS3Utils().writeFile(bucketName, path, contents);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String stripLeadingSlash(String path)
+   {
+      if(path == null)
+      {
+         return (null);
+      }
+      return (path.replaceFirst("^/+", ""));
+   }
+
+
+
+   /*******************************************************************************
+    ** Get a string that represents the full path to a file.
+    *******************************************************************************/
+   @Override
+   public String getFullPathForFile(S3ObjectSummary s3ObjectSummary)
+   {
+      return (s3ObjectSummary.getKey());
+   }
+
+
+
+   /*******************************************************************************
+    ** e.g., with a base path of /foo/
+    ** and a table path of /bar/
+    ** and a file at /foo/bar/baz.txt
+    ** give us just the baz.txt part.
+    *******************************************************************************/
+   @Override
+   public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table)
+   {
+      String tablePath    = getFullBasePath(table, backend);
+      String strippedPath = filePath.replaceFirst("^/*" + stripLeadingSlash(tablePath), "");
+      return (strippedPath);
+   }
+
+
+
+   /*******************************************************************************
+    ** In contrast with the DeleteAction, which deletes RECORDS - this is a
+    ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
+    ** e.g., for post-ETL.
+    **
+    ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
+    *******************************************************************************/
+   @Override
+   public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
+   {
+      QBackendMetaData backend    = instance.getBackend(table.getBackendName());
+      String           bucketName = ((S3BackendMetaData) backend).getBucketName();
+
+      getS3Utils().deleteObject(bucketName, fileReference);
+   }
+
+
+
+   /*******************************************************************************
+    ** In contrast with the DeleteAction, which deletes RECORDS - this is a
+    ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
+    ** e.g., for post-ETL.
+    **
+    ** @param destination assumed to be a file path - not a directory
+    ** @throws FilesystemException if the move is known to have failed
+    *******************************************************************************/
+   @Override
+   public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException
+   {
+      QBackendMetaData backend    = instance.getBackend(table.getBackendName());
+      String           bucketName = ((S3BackendMetaData) backend).getBucketName();
+      destination = stripLeadingSlash(stripDuplicatedSlashes(destination));
+
+      getS3Utils().moveObject(bucketName, source, destination);
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java

@@ -0,0 +1,60 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
+import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
+import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
+import org.apache.commons.lang.NotImplementedException;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class S3DeleteAction implements DeleteInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public DeleteResult execute(DeleteRequest deleteRequest) throws QException
+   {
+      throw new NotImplementedException("S3 delete not implemented");
+      /*
+      try
+      {
+         DeleteResult rs = new DeleteResult();
+         QTableMetaData table = deleteRequest.getTable();
+
+
+         // return rs;
+      }
+      catch(Exception e)
+      {
+         throw new QException("Error executing delete: " + e.getMessage(), e);
+      }
+      */
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java

@@ -0,0 +1,63 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
+import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
+import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
+import org.apache.commons.lang.NotImplementedException;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class S3InsertAction implements InsertInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public InsertResult execute(InsertRequest insertRequest) throws QException
+   {
+      throw new NotImplementedException("S3 insert not implemented");
+      /*
+      try
+      {
+         InsertResult rs = new InsertResult();
+         QTableMetaData table = insertRequest.getTable();
+
+         List<QRecord> recordsWithStatus = new ArrayList<>();
+         rs.setRecords(recordsWithStatus);
+
+
+         // return rs;
+      }
+      catch(Exception e)
+      {
+         throw new QException("Error executing insert: " + e.getMessage(), e);
+      }
+      */
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryAction.java

@@ -0,0 +1,45 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
+import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
+import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class S3QueryAction extends AbstractS3Action implements QueryInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QueryResult execute(QueryRequest queryRequest) throws QException
+   {
+      return (super.executeQuery(queryRequest));
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java

@@ -0,0 +1,62 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
+import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
+import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
+import org.apache.commons.lang.NotImplementedException;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class S3UpdateAction implements UpdateInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public UpdateResult execute(UpdateRequest updateRequest) throws QException
+   {
+      throw new NotImplementedException("S3 update not implemented");
+      /*
+      try
+      {
+         UpdateResult rs = new UpdateResult();
+         QTableMetaData table = updateRequest.getTable();
+
+         List<QRecord> records = new ArrayList<>();
+         rs.setRecords(records);
+
+         // return rs;
+      }
+      catch(Exception e)
+      {
+         throw new QException("Error executing update: " + e.getMessage(), e);
+      }
+      */
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java

@@ -0,0 +1,207 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
+
+
+import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
+
+
+/*******************************************************************************
+ ** S3 backend meta data.
+ *******************************************************************************/
+public class S3BackendMetaData extends AbstractFilesystemBackendMetaData
+{
+   private String bucketName;
+   private String accessKey;
+   private String secretKey;
+   private String region;
+
+
+
+   /*******************************************************************************
+    ** Default Constructor.
+    *******************************************************************************/
+   public S3BackendMetaData()
+   {
+      super();
+      setBackendType(S3BackendModule.class);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for bucketName
+    **
+    *******************************************************************************/
+   public String getBucketName()
+   {
+      return bucketName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for bucketName
+    **
+    *******************************************************************************/
+   public void setBucketName(String bucketName)
+   {
+      this.bucketName = bucketName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for bucketName
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public <T extends S3BackendMetaData> T withBucketName(String bucketName)
+   {
+      this.bucketName = bucketName;
+      return (T) this;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for accessKey
+    **
+    *******************************************************************************/
+   public String getAccessKey()
+   {
+      return accessKey;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for accessKey
+    **
+    *******************************************************************************/
+   public void setAccessKey(String accessKey)
+   {
+      this.accessKey = accessKey;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for accessKey
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public <T extends S3BackendMetaData> T withAccessKey(String accessKey)
+   {
+      this.accessKey = accessKey;
+      return (T) this;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for secretKey
+    **
+    *******************************************************************************/
+   public String getSecretKey()
+   {
+      return secretKey;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for secretKey
+    **
+    *******************************************************************************/
+   public void setSecretKey(String secretKey)
+   {
+      this.secretKey = secretKey;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for secretKey
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public <T extends S3BackendMetaData> T withSecretKey(String secretKey)
+   {
+      this.secretKey = secretKey;
+      return (T) this;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for region
+    **
+    *******************************************************************************/
+   public String getRegion()
+   {
+      return region;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for region
+    **
+    *******************************************************************************/
+   public void setRegion(String region)
+   {
+      this.region = region;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for region
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public <T extends S3BackendMetaData> T withRegion(String region)
+   {
+      this.region = region;
+      return (T) this;
+   }
+
+
+
+   /*******************************************************************************
+    ** Called by the QInstanceEnricher - to do backend-type-specific enrichments.
+    ** Original use case is:  reading secrets into fields (e.g., passwords).
+    ** TODO - migrate to use @InterpretableFields (and complete that impl on core side)
+    *******************************************************************************/
+   @Override
+   public void enrich()
+   {
+      super.enrich();
+      QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
+      accessKey = interpreter.interpret(accessKey);
+      secretKey = interpreter.interpret(secretKey);
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3TableBackendDetails.java

@@ -0,0 +1,46 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
+
+
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
+import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
+
+
+/*******************************************************************************
+ ** S3 specific Extension of QTableBackendDetails
+ *******************************************************************************/
+public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails
+{
+
+
+
+   /*******************************************************************************
+    ** Default Constructor.
+    *******************************************************************************/
+   public S3TableBackendDetails()
+   {
+      super();
+      setBackendType(S3BackendModule.class);
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java

@@ -0,0 +1,249 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.util.ArrayList;
+import java.util.List;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.ListObjectsV2Request;
+import com.amazonaws.services.s3.model.ListObjectsV2Result;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
+import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+
+/*******************************************************************************
+ ** Utility methods for working with AWS S3.
+ **
+ ** Note:  May need a constructor (or similar) in the future that takes the
+ ** S3BackendMetaData - e.g., if we need some metaData to construct the AmazonS3
+ ** (api client) object, such as region, or authentication.
+ *******************************************************************************/
+public class S3Utils
+{
+   private static final Logger LOG = LogManager.getLogger(S3Utils.class);
+
+   private AmazonS3 amazonS3;
+
+
+
+   /*******************************************************************************
+    ** List the objects in an S3 bucket matching a glob, per:
+    ** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
+    *******************************************************************************/
+   public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob)
+   {
+      //////////////////////////////////////////////////////////////////////////////////////////////////
+      // s3 list requests find nothing if the path starts with a /, so strip away any leading slashes //
+      // also strip away trailing /'s, for consistent known paths.                                    //
+      // also normalize any duplicated /'s to a single /.                                             //
+      //////////////////////////////////////////////////////////////////////////////////////////////////
+      path = path.replaceFirst("^/+", "").replaceFirst("/+$", "").replaceAll("//+", "/");
+      String prefix = path;
+
+      // todo - maybe this is some error - that the user put a * in the path instead of the glob?
+      if(prefix.indexOf('*') > -1)
+      {
+         prefix = prefix.substring(0, prefix.indexOf('*'));
+      }
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob.                //
+      // That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. //
+      // Anyway, add a trailing /* to globs with no wildcards (or just a '*' if it's a request for the root (""))       //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(glob == null)
+      {
+         glob = "";
+      }
+      if(!glob.contains("*"))
+      {
+         if(glob.equals(""))
+         {
+            glob += "*";
+         }
+         else
+         {
+            glob += "/*";
+         }
+      }
+
+      String pathMatcherArg = AbstractFilesystemAction.stripDuplicatedSlashes("glob:/" + path + "/" + glob);
+      PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher(pathMatcherArg);
+
+      ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request()
+         .withBucketName(bucketName)
+         .withPrefix(prefix);
+
+      ListObjectsV2Result   listObjectsV2Result = null;
+      List<S3ObjectSummary> rs                  = new ArrayList<>();
+
+      do
+      {
+         if(listObjectsV2Result != null)
+         {
+            listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken());
+         }
+         LOG.info("Listing bucket=" + bucketName + ", path=" + path);
+         listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request);
+
+         //////////////////////////////////
+         // put files in the result list //
+         //////////////////////////////////
+         for(S3ObjectSummary objectSummary : listObjectsV2Result.getObjectSummaries())
+         {
+            String key = objectSummary.getKey();
+
+            //////////////////////////////////////////////////////////////////////////////////////////////////////
+            // it looks like keys in s3 can have duplicated /'s - so normalize those, to create a "sane" result //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////
+            key = key.replaceAll("//+", "/");
+
+            ////////////////////////////////////////////////////////////////////////////////
+            // always skip folders                                                        //
+            // this seemed to fire when it was first written, but not in our unit tests - //
+            // is this a difference with real s3 vs. localstack possibly?                 //
+            ////////////////////////////////////////////////////////////////////////////////
+            if(key.endsWith("/"))
+            {
+               LOG.debug("Skipping file [{}] because it is a folder", key);
+               continue;
+            }
+
+            ///////////////////////////////////////////
+            // skip files that do not match the glob //
+            ///////////////////////////////////////////
+            if(!pathMatcher.matches(Path.of(URI.create("file:///" + key))))
+            {
+               LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob);
+               continue;
+            }
+
+            rs.add(objectSummary);
+         }
+      }
+      while(listObjectsV2Result.isTruncated());
+
+      return rs;
+   }
+
+
+
+   /*******************************************************************************
+    ** Get the contents (as an InputStream) for an object in s3
+    *******************************************************************************/
+   public InputStream getObjectAsInputStream(S3ObjectSummary s3ObjectSummary)
+   {
+      return getAmazonS3().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()).getObjectContent();
+   }
+
+
+
+   /*******************************************************************************
+    ** Write a file
+    *******************************************************************************/
+   public void writeFile(String bucket, String key, byte[] contents)
+   {
+      ObjectMetadata objectMetadata = new ObjectMetadata();
+      objectMetadata.setContentLength(contents.length);
+      getAmazonS3().putObject(bucket, key, new ByteArrayInputStream(contents), objectMetadata);
+   }
+
+
+
+   /*******************************************************************************
+    ** Delete an object (file) from a bucket
+    *******************************************************************************/
+   public void deleteObject(String bucketName, String key) throws FilesystemException
+   {
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // note, aws s3 api does not appear to have any way to check the success or failure here... //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      try
+      {
+         getAmazonS3().deleteObject(bucketName, key);
+      }
+      catch(Exception e)
+      {
+         throw (new FilesystemException("Error deleting s3 object " + key + " in bucket " + bucketName, e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Move an object (file) within a bucket
+    *******************************************************************************/
+   public void moveObject(String bucketName, String source, String destination) throws FilesystemException
+   {
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // note, aws s3 api does not appear to have any way to check the success or failure here... //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      try
+      {
+         getAmazonS3().copyObject(bucketName, source, bucketName, destination);
+         getAmazonS3().deleteObject(bucketName, source);
+      }
+      catch(Exception e)
+      {
+         throw (new FilesystemException("Error moving s3 object " + source + " to " + destination + " in bucket " + bucketName, e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for AmazonS3 client object.
+    *******************************************************************************/
+   public void setAmazonS3(AmazonS3 amazonS3)
+   {
+      this.amazonS3 = amazonS3;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for AmazonS3 client object.
+    *******************************************************************************/
+   public AmazonS3 getAmazonS3()
+   {
+      if(amazonS3 == null)
+      {
+         // TODO - get this (and other props?) from backend meta data
+         amazonS3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build();
+      }
+
+      return amazonS3;
+   }
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java

@@ -0,0 +1,227 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem;
+
+
+import java.io.File;
+import java.io.IOException;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.modules.mock.MockAuthenticationModule;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
+import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
+import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
+import org.apache.commons.io.FileUtils;
+
+
+/*******************************************************************************
+ ** Utility methods for Filesystem module tests
+ *******************************************************************************/
+public class TestUtils
+{
+   public static final String BACKEND_NAME_LOCAL_FS      = "local-filesystem";
+   public static final String BACKEND_NAME_S3            = "s3";
+   public static final String TABLE_NAME_PERSON_LOCAL_FS = "person";
+   public static final String TABLE_NAME_PERSON_S3       = "person-s3";
+
+   ///////////////////////////////////////////////////////////////////
+   // shouldn't be accessed directly, as we append a counter to it. //
+   ///////////////////////////////////////////////////////////////////
+   public static final String BASE_PATH = "/tmp/filesystem-tests";
+
+   //////////////////////////////////////////////////////////////////////////////
+   // Used to make each test method have a unique folder path, more or less... //
+   // See methods that work with it.                                           //
+   //////////////////////////////////////////////////////////////////////////////
+   private static int testInstanceCounter = 0;
+
+
+
+   /*******************************************************************************
+    ** Meant to be called in a @BeforeEach - increment an internal counter that will
+    ** give us a unique directory name for each test method.
+    *******************************************************************************/
+   public static void increaseTestInstanceCounter()
+   {
+      testInstanceCounter++;
+   }
+
+
+
+   /*******************************************************************************
+    ** Reset the counter to 0 (e.g., to let some tests have a known value).
+    *******************************************************************************/
+   public static void resetTestInstanceCounter()
+   {
+      testInstanceCounter = 0;
+   }
+
+
+
+   /*******************************************************************************
+    ** Get the current value of the testInstanceCounter.  See {@link #increaseTestInstanceCounter()}
+    *******************************************************************************/
+   public static int getTestInstanceCounter()
+   {
+      return (testInstanceCounter);
+   }
+
+
+
+   /*******************************************************************************
+    ** Meant to run both after and before test methods - makes sure the file system
+    ** is empty for the path under the instance.
+    *******************************************************************************/
+   public static void cleanInstanceFiles() throws IOException
+   {
+      FileUtils.deleteDirectory(new File(BASE_PATH + File.separator + testInstanceCounter));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QInstance defineInstance() throws QInstanceValidationException
+   {
+      QInstance qInstance = new QInstance();
+      qInstance.setAuthentication(defineAuthentication());
+      qInstance.addBackend(defineLocalFilesystemBackend());
+      qInstance.addTable(defineLocalFilesystemJSONPersonTable());
+      qInstance.addBackend(defineS3Backend());
+      qInstance.addTable(defineS3CSVPersonTable());
+
+      new QInstanceValidator().validate(qInstance);
+
+      return (qInstance);
+   }
+
+
+
+   /*******************************************************************************
+    ** Define the authentication used in standard tests - using 'mock' type.
+    **
+    *******************************************************************************/
+   public static QAuthenticationMetaData defineAuthentication()
+   {
+      return new QAuthenticationMetaData()
+         .withName("mock")
+         .withType("mock");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static FilesystemBackendMetaData defineLocalFilesystemBackend()
+   {
+      return (new FilesystemBackendMetaData()
+         .withBasePath(BASE_PATH + File.separator + testInstanceCounter)
+         .withName(BACKEND_NAME_LOCAL_FS));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QTableMetaData defineLocalFilesystemJSONPersonTable()
+   {
+      return new QTableMetaData()
+         .withName(TABLE_NAME_PERSON_LOCAL_FS)
+         .withLabel("Person")
+         .withBackendName(defineLocalFilesystemBackend().getName())
+         .withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"))
+         .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"))
+         .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"))
+         .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
+         .withField(new QFieldMetaData("email", QFieldType.STRING))
+         .withBackendDetails(new FilesystemTableBackendDetails()
+            .withBasePath("persons")
+            .withRecordFormat(RecordFormat.JSON)
+            .withCardinality(Cardinality.MANY)
+         );
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static S3BackendMetaData defineS3Backend()
+   {
+      return (new S3BackendMetaData()
+         .withBucketName(BaseS3Test.BUCKET_NAME)
+         .withBasePath(BaseS3Test.TEST_FOLDER)
+         .withName(BACKEND_NAME_S3));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QTableMetaData defineS3CSVPersonTable()
+   {
+      return new QTableMetaData()
+         .withName(TABLE_NAME_PERSON_S3)
+         .withLabel("Person S3 Table")
+         .withBackendName(defineS3Backend().getName())
+         .withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"))
+         .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"))
+         .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"))
+         .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
+         .withField(new QFieldMetaData("email", QFieldType.STRING))
+         .withBackendDetails(new S3TableBackendDetails()
+            .withRecordFormat(RecordFormat.CSV)
+            .withCardinality(Cardinality.MANY)
+         );
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QSession getMockSession()
+   {
+      MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule();
+      return (mockAuthenticationModule.createSession(null));
+   }
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java

@@ -0,0 +1,175 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.local;
+
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
+import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
+import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+
+/*******************************************************************************
+ ** Unit test for FilesystemBackendModule
+ *******************************************************************************/
+public class FilesystemBackendModuleTest
+{
+   private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist";
+
+
+
+   @BeforeEach
+   public void beforeEach() throws IOException
+   {
+      new FilesystemActionTest().primeFilesystem();
+   }
+
+
+
+   @AfterEach
+   public void afterEach() throws Exception
+   {
+      new FilesystemActionTest().cleanFilesystem();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testDeleteFile() throws Exception
+   {
+      QInstance      qInstance = TestUtils.defineInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
+
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // first list the files - then delete one, then re-list, and assert that we have one fewer //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      List<File> filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+
+      FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
+      filesystemBackendModule.getActionBase().deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath());
+
+      List<File> filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+      Assertions.assertEquals(filesBeforeDelete.size() - 1, filesAfterDelete.size(),
+         "Should be one fewer file listed after deleting one.");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testDeleteFileDoesNotExist() throws Exception
+   {
+      QInstance      qInstance = TestUtils.defineInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
+      // note, our contract is that as long as the file doesn't exist after calling delete (e.g., if it wasn't there //
+      // to begin with, then we're okay, and don't expect an exception                                               //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      List<File> filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+
+      FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
+      filesystemBackendModule.getActionBase().deleteFile(qInstance, table, PATH_THAT_WONT_EXIST);
+
+      List<File> filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+      Assertions.assertEquals(filesBeforeDelete.size(), filesAfterDelete.size(),
+         "Should be same number of files after deleting bogus path");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testMoveFile() throws Exception
+   {
+      QInstance      qInstance = TestUtils.defineInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
+      String         basePath  = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
+      String         subPath   = basePath + File.separator + "subdir";
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      // first list the files (non-recursively) - then move one into a sub-folder, then re-list, and    //
+      // assert that we have one fewer then list again including sub-folders, and see the changed count //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      List<File> filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+
+      FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
+      String                  originalFilePath        = filesBeforeMove.get(0).getAbsolutePath();
+      String                  movedFilePath           = originalFilePath.replace(basePath, subPath);
+
+      filesystemBackendModule.getActionBase().moveFile(qInstance, table, originalFilePath, movedFilePath);
+
+      List<File> filesAfterMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+      Assertions.assertEquals(filesBeforeMove.size() - 1, filesAfterMove.size(), "Should be one fewer file in the listing after moving one.");
+
+      //////////////////////////////////////////////////////////////////////////
+      // move the file back and assert that the count goes back to the before //
+      //////////////////////////////////////////////////////////////////////////
+      filesystemBackendModule.getActionBase().moveFile(qInstance, table, movedFilePath, originalFilePath);
+
+      List<File> filesAfterMoveBack = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+      Assertions.assertEquals(filesBeforeMove.size(), filesAfterMoveBack.size(), "Should be original number of files after moving back");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testMoveFileDoesNotExit() throws Exception
+   {
+      QInstance      qInstance        = TestUtils.defineInstance();
+      QTableMetaData table            = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
+      String         basePath         = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
+      String         subPath          = basePath + File.separator + "subdir";
+      List<File>     filesBeforeMove  = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+      String         originalFilePath = filesBeforeMove.get(0).getAbsolutePath();
+      String         movedFilePath    = originalFilePath.replace(basePath, subPath);
+
+      Assertions.assertThrows(FilesystemException.class, () ->
+      {
+         FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
+         filesystemBackendModule.getActionBase().moveFile(qInstance, table, PATH_THAT_WONT_EXIST, movedFilePath);
+      });
+
+   }
+
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java

@@ -0,0 +1,128 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.local.actions;
+
+
+import java.io.File;
+import java.io.IOException;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
+import org.apache.commons.io.FileUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import static org.junit.jupiter.api.Assertions.fail;
+
+
+/*******************************************************************************
+ ** Base class for Filesystem action tests.
+ **
+ ** Knows how to set up the filesystem for the tests.
+ *******************************************************************************/
+public class FilesystemActionTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   public void beforeEach() throws Exception
+   {
+      primeFilesystem();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @AfterEach
+   public void afterEach() throws Exception
+   {
+      cleanFilesystem();
+   }
+
+
+
+   /*******************************************************************************
+    ** Set up the file system
+    *******************************************************************************/
+   public void primeFilesystem() throws IOException
+   {
+      TestUtils.cleanInstanceFiles();
+      TestUtils.increaseTestInstanceCounter();
+      FilesystemBackendMetaData filesystemBackendMetaData = TestUtils.defineLocalFilesystemBackend();
+
+      File    baseDirectory = new File(filesystemBackendMetaData.getBasePath());
+      boolean mkdirsResult  = baseDirectory.mkdirs();
+      if(!mkdirsResult)
+      {
+         fail("Failed to make directories at [" + baseDirectory + "] for filesystem backend module");
+      }
+
+      writePersonFiles(baseDirectory);
+   }
+
+
+
+   /*******************************************************************************
+    ** Write some data files into the directory for the filesystem module.
+    *******************************************************************************/
+   private void writePersonFiles(File baseDirectory) throws IOException
+   {
+      String fullPath = baseDirectory.getAbsolutePath();
+      if (TestUtils.defineLocalFilesystemJSONPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
+      {
+         if (StringUtils.hasContent(details.getBasePath()))
+         {
+            fullPath += File.separatorChar + details.getBasePath();
+         }
+      }
+      fullPath += File.separatorChar;
+
+      String jsonData1 = """
+         [
+            {"id":1,"createDate":"2021-10-26 14:39:37","modifyDate":"2021-10-26 14:39:37","firstName":"John","lastName":"Doe","birthDate":"1981-01-01","email":"john@kingsrook.com"},
+            {"id":2,"createDate":"2022-06-17 14:52:59","modifyDate":"2022-06-17 14:52:59","firstName":"Jane","lastName":"Smith","birthDate":"1982-02-02","email":"jane@kingsrook.com"}
+         ]
+         """;
+      FileUtils.writeStringToFile(new File(fullPath + "DATA-1.json"), jsonData1);
+
+      String jsonData2 = """
+         [
+            {"id":3,"createDate":"2021-11-27 15:40:38","modifyDate":"2021-11-27 15:40:38","firstName":"Homer","lastName":"S","birthDate":"1983-03-03","email":"homer.s@kingsrook.com"}
+         ]
+         """;
+      FileUtils.writeStringToFile(new File(fullPath + "DATA-2.json"), jsonData2);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void cleanFilesystem() throws IOException
+   {
+      TestUtils.cleanInstanceFiles();
+   }
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java

@@ -0,0 +1,68 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.local.actions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
+import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class FilesystemQueryActionTest extends FilesystemActionTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testQuery1() throws QException
+   {
+      QueryRequest queryRequest = initQueryRequest();
+      QueryResult  queryResult  = new FilesystemQueryAction().execute(queryRequest);
+      Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows");
+      Assertions.assertTrue(queryResult.getRecords().stream()
+            .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
+         "All records should have a full-path in their backend details, matching the test folder name");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QueryRequest initQueryRequest() throws QInstanceValidationException
+   {
+      QueryRequest queryRequest = new QueryRequest();
+      queryRequest.setInstance(TestUtils.defineInstance());
+      queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
+      return queryRequest;
+   }
+
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java

@@ -0,0 +1,77 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata;
+
+
+import java.io.IOException;
+import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for FilesystemBackendMetaData
+ *******************************************************************************/
+class FilesystemBackendMetaDataTest
+{
+
+
+   /*******************************************************************************
+    ** Test that an instance can be serialized as expected
+    *******************************************************************************/
+   @Test
+   public void testSerializingToJson() throws QInstanceValidationException
+   {
+      TestUtils.resetTestInstanceCounter();
+      QInstance qInstance = TestUtils.defineInstance();
+      String    json      = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance);
+      System.out.println(JsonUtils.prettyPrint(json));
+      System.out.println(json);
+      String expectToContain = """
+         "local-filesystem":{"basePath":"/tmp/filesystem-tests/0","backendType":"filesystem","name":"local-filesystem"}""";
+      assertTrue(json.contains(expectToContain));
+   }
+
+
+
+   /*******************************************************************************
+    ** Test that an instance can be deserialized as expected
+    *******************************************************************************/
+   @Test
+   public void testDeserializingFromJson() throws IOException, QInstanceValidationException
+   {
+      QInstanceAdapter qInstanceAdapter = new QInstanceAdapter();
+
+      QInstance qInstance = TestUtils.defineInstance();
+      String    json      = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance);
+
+      QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
+      assertThat(deserialized).usingRecursiveComparison()
+         .ignoringFields("hasBeenValidated") // note, this field is @JsonIgnore
+         .isEqualTo(qInstance);
+   }
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java

@@ -0,0 +1,228 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
+
+
+import java.io.File;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
+import org.apache.commons.io.FileUtils;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for BasicETLCleanupSourceFilesFunction
+ *******************************************************************************/
+public class BasicETLCleanupSourceFilesFunctionTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testDelete1Record1File() throws Exception
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+      String    filePath  = getRandomFilePathPersonTable(qInstance);
+      testDelete(qInstance, List.of(filePath));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testDelete2Records1File() throws Exception
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+      String    filePath  = getRandomFilePathPersonTable(qInstance);
+      testDelete(qInstance, List.of(filePath, filePath));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testDelete2Record2File() throws Exception
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+      String    filePath1 = getRandomFilePathPersonTable(qInstance);
+      String    filePath2 = getRandomFilePathPersonTable(qInstance);
+      testDelete(qInstance, List.of(filePath1, filePath2));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testMove1Record1File() throws Exception
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+      String    filePath  = getRandomFilePathPersonTable(qInstance);
+      testMove(qInstance, List.of(filePath));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testMove2Records1File() throws Exception
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+      String    filePath  = getRandomFilePathPersonTable(qInstance);
+      testMove(qInstance, List.of(filePath, filePath));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testMove2Record2File() throws Exception
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+      String    filePath1 = getRandomFilePathPersonTable(qInstance);
+      String    filePath2 = getRandomFilePathPersonTable(qInstance);
+      testMove(qInstance, List.of(filePath1, filePath2));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void testDelete(QInstance qInstance, List<String> filePaths) throws Exception
+   {
+      RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of(
+         BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_DELETE,
+         // todo - even though this field isn't needed, since we gave a value of "delete"
+         //  the RunFunctionAction considers any missing input to be an error...
+         BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, ""));
+
+      assertNull(runFunctionResult.getError());
+      for(String filePath : filePaths)
+      {
+         assertFalse(new File(filePath).exists(), "File should have been deleted.");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void testMove(QInstance qInstance, List<String> filePaths) throws Exception
+   {
+      String trashDir = File.separator + "tmp" + File.separator + "trash";
+      RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of(
+         BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_MOVE,
+         BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir));
+
+      assertNull(runFunctionResult.getError());
+
+      for(String filePath : filePaths)
+      {
+         assertFalse(new File(filePath).exists(), "File should have been moved.");
+
+         String movedPath = trashDir + File.separator + (new File(filePath).getName());
+         assertTrue(new File(movedPath).exists(), "File should have been moved.");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private RunFunctionResult runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception
+   {
+      QFunctionMetaData qFunctionMetaData = new BasicETLCleanupSourceFilesFunction().defineFunctionMetaData();
+      QProcessMetaData  qProcessMetaData  = new QProcessMetaData().withName("testScaffold").addFunction(qFunctionMetaData);
+      qInstance.addProcess(qProcessMetaData);
+
+      HashSet<String> filePathsSet = new HashSet<>(filePaths);
+      for(String filePath : filePathsSet)
+      {
+         File file = new File(filePath);
+         FileUtils.writeStringToFile(file, "content");
+      }
+
+      // List<QRecord> records = filePaths.stream()
+      //    .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList();
+
+      RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
+      runFunctionRequest.setFunctionName(qFunctionMetaData.getName());
+      runFunctionRequest.setProcessName(qProcessMetaData.getName());
+      // runFunctionRequest.setRecords(records);
+      runFunctionRequest.setSession(TestUtils.getMockSession());
+      runFunctionRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
+      runFunctionRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
+      runFunctionRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
+
+      for(Map.Entry<String, String> entry : values.entrySet())
+      {
+         runFunctionRequest.addValue(entry.getKey(), entry.getValue());
+      }
+
+      RunFunctionAction runFunctionAction = new RunFunctionAction();
+      return (runFunctionAction.execute(runFunctionRequest));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String getRandomFilePathPersonTable(QInstance qInstance)
+   {
+      FilesystemBackendMetaData     backend        = (FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
+      FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS).getBackendDetails();
+      String                        tablePath      = backend.getBasePath() + File.separator + backendDetails.getBasePath();
+      String                        filePath       = tablePath + File.separator + UUID.randomUUID();
+      return filePath;
+   }
+
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java

@@ -0,0 +1,266 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync;
+
+
+import java.util.List;
+import java.util.stream.Collectors;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
+import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
+import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
+import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModuleSubclassForTest;
+import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
+import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for FilesystemSyncProcess using S3 backend
+ *******************************************************************************/
+class FilesystemSyncProcessS3Test extends BaseS3Test
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test() throws Exception
+   {
+      QBackendModuleDispatcher.registerBackendModule(new S3BackendModuleSubclassForTest());
+      QInstance qInstance = TestUtils.defineInstance();
+
+      String sourceBucket     = "source-bucket";
+      String archiveBucket    = "archive-bucket";
+      String processingBucket = "processing-bucket";
+
+      getAmazonS3().createBucket(sourceBucket);
+      getAmazonS3().createBucket(archiveBucket);
+      getAmazonS3().createBucket(processingBucket);
+
+      S3BackendMetaData sourceBackend     = defineBackend(qInstance, "source", sourceBucket);
+      S3BackendMetaData archiveBackend    = defineBackend(qInstance, "archive", archiveBucket);
+      S3BackendMetaData processingBackend = defineBackend(qInstance, "processing", processingBucket);
+
+      QTableMetaData sourceTable     = defineTable(qInstance, "source", sourceBackend, "source", "*/l3/*.csv");
+      QTableMetaData archiveTable    = defineTable(qInstance, "archive", archiveBackend, "archive", "*/l3/*.csv");
+      QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv");
+
+      QProcessMetaData  process  = new FilesystemSyncProcess().defineProcessMetaData();
+      QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
+      qInstance.addProcess(process);
+
+      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
+      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
+      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
+
+      ///////////////////////////
+      // write some test files //
+      ///////////////////////////
+      writeTestFile(sourceBackend, sourceTable, "foo/l3/1.csv", "x");
+      writeTestFile(sourceBackend, sourceTable, "bar/l3/2.csv", "x");
+      writeTestFile(archiveBackend, archiveTable, "foo/l3/1.csv", "x");
+
+      printTableListing(sourceBackend, sourceTable);
+      printTableListing(archiveBackend, archiveTable);
+      printTableListing(processingBackend, processingTable);
+
+      //////////////////////
+      // run the function //
+      //////////////////////
+      RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
+      runFunctionRequest.setFunctionName(function.getName());
+      runFunctionRequest.setProcessName(process.getName());
+      runFunctionRequest.setSession(TestUtils.getMockSession());
+
+      RunFunctionAction runFunctionAction = new RunFunctionAction();
+      RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
+      System.out.println(runFunctionResult);
+
+      printTableListing(sourceBackend, sourceTable);
+      printTableListing(archiveBackend, archiveTable);
+      printTableListing(processingBackend, processingTable);
+
+      assertTableListing(archiveBackend, archiveTable, "root/archive/foo/l3/1.csv", "root/archive/bar/l3/2.csv");
+      assertTableListing(processingBackend, processingTable, "root/processing/bar/l3/2.csv");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testArchiveAndProcessingInSameBucket() throws Exception
+   {
+      QBackendModuleDispatcher.registerBackendModule(new S3BackendModuleSubclassForTest());
+      QInstance qInstance = TestUtils.defineInstance();
+
+      String vendorBucket = "vendor-bucket";
+      String localBucket  = "local-bucket";
+
+      getAmazonS3().createBucket(vendorBucket);
+      getAmazonS3().createBucket(localBucket);
+
+      S3BackendMetaData vendorBackend = defineBackend(qInstance, "source", vendorBucket);
+      S3BackendMetaData localBackend  = defineBackend(qInstance, "archive", localBucket);
+
+      QTableMetaData sourceTable     = defineTable(qInstance, "source", vendorBackend, "source", "*/l3/*.csv");
+      QTableMetaData archiveTable    = defineTable(qInstance, "archive", localBackend, "archive", "*/l3/*.csv");
+      QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv");
+
+      QProcessMetaData  process  = new FilesystemSyncProcess().defineProcessMetaData();
+      QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
+      qInstance.addProcess(process);
+
+      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
+      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
+      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
+
+      ///////////////////////////
+      // write some test files //
+      ///////////////////////////
+      writeTestFile(vendorBackend, sourceTable, "foo/l3/1.csv", "x");
+      writeTestFile(vendorBackend, sourceTable, "bar/l3/2.csv", "x");
+      writeTestFile(localBackend, archiveTable, "foo/l3/1.csv", "x");
+
+      printTableListing(vendorBackend, sourceTable);
+      printTableListing(localBackend, archiveTable);
+      printTableListing(localBackend, processingTable);
+
+      //////////////////////
+      // run the function //
+      //////////////////////
+      RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
+      runFunctionRequest.setFunctionName(function.getName());
+      runFunctionRequest.setProcessName(process.getName());
+      runFunctionRequest.setSession(TestUtils.getMockSession());
+
+      RunFunctionAction runFunctionAction = new RunFunctionAction();
+      RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
+      System.out.println(runFunctionResult);
+
+      printTableListing(vendorBackend, sourceTable);
+      printTableListing(localBackend, archiveTable);
+      printTableListing(localBackend, processingTable);
+
+      assertTableListing(localBackend, archiveTable, "root/archive/foo/l3/1.csv", "root/archive/bar/l3/2.csv");
+      assertTableListing(localBackend, processingTable, "root/processing/bar/l3/2.csv");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException
+   {
+      S3BackendModule  module     = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
+      AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
+
+      List<S3ObjectSummary> s3ObjectSummaries = actionBase.listFiles(table, backend);
+      assertEquals(paths.length, s3ObjectSummaries.size(), "Expected number of files in table: " + table.getName());
+      for(String path : paths)
+      {
+         assertTrue(s3ObjectSummaries.stream().anyMatch(s3o -> s3o.getKey().equals(path)),
+            "Path [" + path + "] should be in the listing, but was not.  Full listing is: " +
+               s3ObjectSummaries.stream().map(S3ObjectSummary::getKey).collect(Collectors.joining(",")));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException
+   {
+      S3BackendModule  module     = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
+      AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
+
+      System.out.println("Files in: " + table.getName());
+      actionBase.listFiles(table, backend).forEach(o -> System.out.println(o.getKey()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void writeTestFile(S3BackendMetaData backend, QTableMetaData table, String name, String content) throws Exception
+   {
+      S3BackendModule  module     = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
+      AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
+      String           fullPath   = actionBase.getFullBasePath(table, backend);
+
+      actionBase.writeFile(backend, fullPath + "/" + name, content.getBytes());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private S3BackendMetaData defineBackend(QInstance qInstance, String which, String bucketName)
+   {
+      QBackendMetaData backendMetaData = new S3BackendMetaData()
+         .withBucketName(bucketName)
+         .withBasePath("root")
+         .withName("backend-" + which);
+      qInstance.addBackend(backendMetaData);
+      return (S3BackendMetaData) backendMetaData;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QTableMetaData defineTable(QInstance qInstance, String which, QBackendMetaData backend, String path, String glob)
+   {
+      QTableMetaData qTableMetaData = new QTableMetaData()
+         .withName("table-" + which)
+         .withBackendName(backend.getName())
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withBackendDetails(new S3TableBackendDetails()
+            .withBasePath(path)
+            .withGlob(glob));
+      qInstance.addTable(qTableMetaData);
+      return (qTableMetaData);
+   }
+
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java

@@ -0,0 +1,123 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync;
+
+
+import java.io.File;
+import java.io.IOException;
+import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
+import org.apache.commons.io.FileUtils;
+import org.junit.jupiter.api.Test;
+
+
+/*******************************************************************************
+ ** Unit test for FilesystemSyncProcess
+ *******************************************************************************/
+class FilesystemSyncProcessTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test() throws Exception
+   {
+      TestUtils.cleanInstanceFiles();
+
+      QTableMetaData    sourceTable     = defineTable("source");
+      QTableMetaData    archiveTable    = defineTable("archive");
+      QTableMetaData    processingTable = defineTable("processing");
+      QProcessMetaData  process         = new FilesystemSyncProcess().defineProcessMetaData();
+      QFunctionMetaData function        = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
+
+      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
+      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
+      function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
+      // function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1);
+
+      QInstance qInstance = TestUtils.defineInstance();
+      qInstance.addTable(sourceTable);
+      qInstance.addTable(archiveTable);
+      qInstance.addTable(processingTable);
+      qInstance.addProcess(process);
+
+      ///////////////////////////
+      // write some test files //
+      ///////////////////////////
+      String basePath = ((FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS)).getBasePath();
+      writeTestFile(basePath, sourceTable, "1.txt", "x");
+      writeTestFile(basePath, sourceTable, "2.txt", "x");
+      // writeTestFile(basePath, sourceTable, "3.txt", "x");
+      writeTestFile(basePath, archiveTable, "2.txt", "x");
+
+      //////////////////////
+      // run the function //
+      //////////////////////
+      RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
+      runFunctionRequest.setFunctionName(function.getName());
+      runFunctionRequest.setProcessName(process.getName());
+      runFunctionRequest.setSession(TestUtils.getMockSession());
+
+      RunFunctionAction runFunctionAction = new RunFunctionAction();
+      RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
+      System.out.println(runFunctionResult);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void writeTestFile(String basePath, QTableMetaData table, String name, String content) throws IOException
+   {
+      String path = ((FilesystemTableBackendDetails) table.getBackendDetails()).getBasePath();
+      File   file = new File(basePath + "/" + path + "/" + name);
+      FileUtils.writeStringToFile(file, content);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QTableMetaData defineTable(String subPath)
+   {
+      return new QTableMetaData()
+         .withName("table-" + subPath)
+         .withBackendName(TestUtils.BACKEND_NAME_LOCAL_FS)
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withBackendDetails(new FilesystemTableBackendDetails()
+            .withBasePath(subPath));
+   }
+
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java

@@ -0,0 +1,162 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3;
+
+
+import cloud.localstack.ServiceName;
+import cloud.localstack.awssdkv1.TestUtils;
+import cloud.localstack.docker.LocalstackDockerExtension;
+import cloud.localstack.docker.annotation.LocalstackDockerProperties;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+
+/*******************************************************************************
+ ** Base class for tests that want to be able to work with localstack s3.
+ *******************************************************************************/
+@ExtendWith(LocalstackDockerExtension.class)
+@LocalstackDockerProperties(services = { ServiceName.S3 }, portEdge = "2960", portElasticSearch = "2961")
+public class BaseS3Test
+{
+   public static final String BUCKET_NAME = "localstack-test-bucket";
+   public static final String TEST_FOLDER = "test-files";
+   public static final String SUB_FOLDER  = "sub-folder";
+
+
+
+   /*******************************************************************************
+    ** Before each unit test, get the test bucket into a known state
+    *******************************************************************************/
+   @BeforeEach
+   public void beforeEach()
+   {
+      AmazonS3 amazonS3 = getAmazonS3();
+
+      amazonS3.createBucket(BUCKET_NAME);
+      amazonS3.putObject(BUCKET_NAME, "0.csv", getCSVHeader());
+      amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/1.csv", getCSVData1());
+      amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2());
+      amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test");
+      amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
+   }
+
+
+
+   /*******************************************************************************
+    ** After each unit test, clean up the bucket
+    *******************************************************************************/
+   @AfterEach
+   public void afterEach()
+   {
+      AmazonS3 amazonS3 = getAmazonS3();
+
+      if(amazonS3.doesBucketExistV2(BUCKET_NAME))
+      {
+         ////////////////////////
+         // todo - paginate... //
+         ////////////////////////
+         for(S3ObjectSummary objectSummary : amazonS3.listObjectsV2(BUCKET_NAME).getObjectSummaries())
+         {
+            amazonS3.deleteObject(BUCKET_NAME, objectSummary.getKey());
+         }
+         amazonS3.deleteBucket(BUCKET_NAME);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Access a localstack-configured s3 client.
+    *******************************************************************************/
+   protected AmazonS3 getAmazonS3()
+   {
+      return (TestUtils.getClientS3());
+   }
+
+
+
+   /*******************************************************************************
+    ** Access the S3Utils object, with localstack-configured s3 client.
+    *******************************************************************************/
+   protected S3Utils getS3Utils()
+   {
+      S3Utils s3Utils = new S3Utils();
+      s3Utils.setAmazonS3(getAmazonS3());
+      return (s3Utils);
+   }
+
+
+
+   /*******************************************************************************
+    ** Access a string of CSV test data.
+    *******************************************************************************/
+   protected String getCSVHeader()
+   {
+      return ("""
+         "id","createDate","modifyDate","firstName","lastName","birthDate","email"
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    ** Access a string of CSV test data.
+    *******************************************************************************/
+   protected String getCSVData1()
+   {
+      return (getCSVHeader() + """
+         "1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com"
+         "2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com"
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    ** Access a string of CSV test data.
+    *******************************************************************************/
+   protected String getCSVData2()
+   {
+      return (getCSVHeader() + """
+         "3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com"
+         "4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com"
+         "5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com"
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    ** Access a string of CSV test data.
+    *******************************************************************************/
+   protected String getCSVData3()
+   {
+      return (getCSVHeader() + """
+         "6","2022-06-20 15:31:02","2022-06-20 15:31:02","Lisa","S","1986-06-06","lisa.s@kingsrook.com"
+         """);
+   }
+
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleSubclassForTest.java

@@ -0,0 +1,53 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3;
+
+
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import cloud.localstack.awssdkv1.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
+import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
+import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
+
+
+/*******************************************************************************
+ ** Subclass of the S3Backend module, meant for use in unit tests, if/where we
+ ** need to make sure we use the localstack version of the S3 client.
+ *******************************************************************************/
+public class S3BackendModuleSubclassForTest extends S3BackendModule
+{
+
+   /*******************************************************************************
+    ** Seed the AbstractS3Action with an s3Utils object that has the localstack
+    ** s3 client in it
+    *******************************************************************************/
+   @Override
+   public AbstractBaseFilesystemAction<S3ObjectSummary> getActionBase()
+   {
+      AbstractS3Action actionBase = (AbstractS3Action) super.getActionBase();
+      S3Utils          s3Utils    = new S3Utils();
+      s3Utils.setAmazonS3(TestUtils.getClientS3());
+      actionBase.setS3Utils(s3Utils);
+      return (actionBase);
+   }
+
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java

@@ -0,0 +1,156 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3;
+
+
+import java.util.List;
+import java.util.UUID;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
+import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+
+/*******************************************************************************
+ ** Unit test for S3BackendModule
+ *******************************************************************************/
+public class S3BackendModuleTest extends BaseS3Test
+{
+   private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testDeleteFile() throws Exception
+   {
+      QInstance      qInstance = TestUtils.defineInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
+
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // first list the files - then delete one, then re-list, and assert that we have one fewer //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      List<S3ObjectSummary> s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
+
+      S3BackendModule  s3BackendModule = new S3BackendModule();
+      AbstractS3Action actionBase      = (AbstractS3Action) s3BackendModule.getActionBase();
+      actionBase.setS3Utils(getS3Utils());
+      actionBase.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey());
+
+      List<S3ObjectSummary> s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
+      Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(),
+         "Should be one fewer file listed after deleting one.");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testDeleteFileDoesNotExist() throws Exception
+   {
+      QInstance      qInstance = TestUtils.defineInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
+      // note, we'd like to detect the non-delete, but there's no such info back from aws it appears?                //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      List<S3ObjectSummary> s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
+
+      S3BackendModule  s3BackendModule = new S3BackendModule();
+      AbstractS3Action actionBase      = (AbstractS3Action) s3BackendModule.getActionBase();
+      actionBase.setS3Utils(getS3Utils());
+      actionBase.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST);
+
+      List<S3ObjectSummary> s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
+      Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size(), s3ObjectSummariesAfterDelete.size(),
+         "Should be same number of files after deleting bogus path");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testMoveFile() throws Exception
+   {
+      QInstance      qInstance = TestUtils.defineInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
+      String         subPath   = TEST_FOLDER + "/" + SUB_FOLDER;
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      // first list the files (non-recursively) - then move one into a sub-folder, then re-list, and    //
+      // assert that we have one fewer then list again including sub-folders, and see the changed count //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      List<S3ObjectSummary> s3ObjectSummariesBeforeMove            = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
+      List<S3ObjectSummary> s3ObjectSummariesInSubFolderBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, subPath, "");
+      List<S3ObjectSummary> s3ObjectSummariesRecursiveBeforeMove   = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**");
+
+      S3BackendModule  s3BackendModule = new S3BackendModule();
+      AbstractS3Action actionBase      = (AbstractS3Action) s3BackendModule.getActionBase();
+      actionBase.setS3Utils(getS3Utils());
+      String key = s3ObjectSummariesBeforeMove.get(0).getKey();
+      actionBase.moveFile(qInstance, table, key, key.replaceFirst(TEST_FOLDER, subPath));
+
+      List<S3ObjectSummary> s3ObjectSummariesAfterMove            = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
+      List<S3ObjectSummary> s3ObjectSummariesRecursiveAfterMove   = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**");
+      List<S3ObjectSummary> s3ObjectSummariesInSubFolderAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, subPath, "");
+
+      Assertions.assertEquals(s3ObjectSummariesBeforeMove.size() - 1, s3ObjectSummariesAfterMove.size(),
+         "Should be one fewer file in the non-recursive listing after moving one.");
+      Assertions.assertEquals(s3ObjectSummariesRecursiveBeforeMove.size(), s3ObjectSummariesRecursiveAfterMove.size(),
+         "Should be same number of files in the recursive listing before and after the move");
+      Assertions.assertEquals(s3ObjectSummariesInSubFolderBeforeMove.size() + 1, s3ObjectSummariesInSubFolderAfterMove.size(),
+         "Should be one move file in the sub-folder listing after moving one.");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testMoveFileDoesNotExit() throws Exception
+   {
+      QInstance      qInstance = TestUtils.defineInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
+      String         subPath   = TEST_FOLDER + "/" + SUB_FOLDER;
+
+      S3BackendModule  s3BackendModule = new S3BackendModule();
+      AbstractS3Action actionBase      = (AbstractS3Action) s3BackendModule.getActionBase();
+      actionBase.setS3Utils(getS3Utils());
+
+      Assertions.assertThrows(FilesystemException.class, () ->
+         actionBase.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, subPath + "/" + UUID.randomUUID())
+      );
+   }
+
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java

@@ -0,0 +1,71 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
+import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
+import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class S3QueryActionTest extends BaseS3Test
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testQuery1() throws QException
+   {
+      QueryRequest  queryRequest  = initQueryRequest();
+      S3QueryAction s3QueryAction = new S3QueryAction();
+      s3QueryAction.setS3Utils(getS3Utils());
+      QueryResult queryResult = s3QueryAction.execute(queryRequest);
+      Assertions.assertEquals(5, queryResult.getRecords().size(), "Expected # of rows from unfiltered query");
+      Assertions.assertTrue(queryResult.getRecords().stream()
+            .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseS3Test.TEST_FOLDER)),
+         "All records should have a full-path in their backend details, matching the test folder name");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QueryRequest initQueryRequest() throws QInstanceValidationException
+   {
+      QueryRequest queryRequest = new QueryRequest();
+      queryRequest.setInstance(TestUtils.defineInstance());
+      queryRequest.setTableName(TestUtils.defineS3CSVPersonTable().getName());
+      return queryRequest;
+   }
+
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java

@@ -0,0 +1,77 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
+
+
+import java.io.IOException;
+import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for S3BackendMetaData
+ *******************************************************************************/
+class S3BackendMetaDataTest
+{
+
+
+   /*******************************************************************************
+    ** Test that an instance can be serialized as expected
+    *******************************************************************************/
+   @Test
+   public void testSerializingToJson() throws QInstanceValidationException
+   {
+      TestUtils.resetTestInstanceCounter();
+      QInstance qInstance = TestUtils.defineInstance();
+      String    json      = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance);
+      System.out.println(JsonUtils.prettyPrint(json));
+      System.out.println(json);
+      String expectToContain = """
+         {"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","secretKey":null,"accessKey":null,"backendType":"s3","name":"s3","region":null}""";
+      assertTrue(json.contains(expectToContain));
+   }
+
+
+
+   /*******************************************************************************
+    ** Test that an instance can be deserialized as expected
+    *******************************************************************************/
+   @Test
+   public void testDeserializingFromJson() throws IOException, QInstanceValidationException
+   {
+      QInstanceAdapter qInstanceAdapter = new QInstanceAdapter();
+
+      QInstance qInstance = TestUtils.defineInstance();
+      String    json      = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance);
+
+      QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
+      assertThat(deserialized).usingRecursiveComparison()
+         .ignoringFields("hasBeenValidated") // note, this field is @JsonIgnore
+         .isEqualTo(qInstance);
+   }
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/PathMatcherGlobTest.java

@@ -0,0 +1,73 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
+
+
+import java.net.URI;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+
+/*******************************************************************************
+ ** Verification for some of the behavior in the S3Utils - working with PathMatcher
+ ** globs.
+ *******************************************************************************/
+public class PathMatcherGlobTest
+{
+
+   @Test
+   public void testPathMatcher() throws Exception
+   {
+      /////////////////////////////////////////////////////////////
+      // note:  must start with for both the pattern and the uri //
+      /////////////////////////////////////////////////////////////
+      PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/*/acme/*/*.csv");
+
+      Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/1234.csv"))), "Glob should match");
+      Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/nj/acme/20220627/1234.csv"))), "Glob should match");
+      Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220628/1234.csv"))), "Glob should match");
+      Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/12345.csv"))), "Glob should match");
+
+      Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/beta/20220627/1234.csv"))), "Glob should not match (beta vs acme)");
+      Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///something/stl/acme/20220627/1234.csv"))), "Glob should not match (wrong start path)");
+      Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/csv"))), "Glob should not match (no file basename)");
+      Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/1234.CSV"))), "Glob should not match (wrong case extension)");
+      Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/extra/1234.csv"))), "Glob should not match (extra dir)");
+      Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/extra/acme/20220627/1234.csv"))), "Glob should not match (extra dir)");
+      Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/extra/stl/acme/20220627/1234.csv"))), "Glob should not match (extra dir)");
+
+      pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/**/acme/*/*.csv");
+      Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/extra/stl/acme/20220627/1234.csv"))), "Glob should match with extra dir");
+      Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/extra/extra2/stl/acme/20220627/1234.csv"))), "Glob should match with 2 extra dirs");
+      Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/acme/20220627/1234.csv"))), "Glob does not match with no dir for **");
+
+      pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/**");
+      Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/1234.csv"))), "Glob should match with extra dir");
+
+      pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/*");
+      Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///1234.csv"))), "Glob should match");
+   }
+
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java

@@ -0,0 +1,84 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
+
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
+import org.apache.commons.io.IOUtils;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class S3UtilsTest extends BaseS3Test
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testListObjectsInBucketAtPath()
+   {
+      S3Utils s3Utils = getS3Utils();
+      assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders");
+      assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders");
+      assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders");
+      assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders");
+      assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
+      assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash");
+      assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash");
+      assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes");
+      assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, "").size(), "Just in the subfolder non-recursive");
+      assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, "/**").size(), "Just in the subfolder recursive");
+      assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "//" + SUB_FOLDER, "/**").size(), "Just in the subfolder recursive, multi /");
+      assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "not-a-real-path/", "").size(), "In a non-existing folder");
+      assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /");
+      assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s");
+      assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string");
+      assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
+      assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testGetObjectAsInputStream() throws IOException
+   {
+      S3Utils               s3Utils           = getS3Utils();
+      List<S3ObjectSummary> s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", "");
+      S3ObjectSummary       s3ObjectSummary   = s3ObjectSummaries.stream().filter(o -> o.getKey().contains("1.csv")).findAny().get();
+      InputStream           inputStream       = s3Utils.getObjectAsInputStream(s3ObjectSummary);
+      String                csvFromS3         = IOUtils.toString(inputStream);
+
+      assertEquals(getCSVData1(), csvFromS3, "File from S3 should match expected content");
+   }
+
+}