QQQ-14, QQQ-16 updated filesystem, s3 implementation, including passable version of basic ETL process & cleanup

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java

@@ -0,0 +1,51 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.base;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
+
+
+/*******************************************************************************
+ ** Interface to add additional functionality commonly among the various filesystem
+ ** module implementations.
+ *******************************************************************************/
+public interface FilesystemBackendModuleInterface
+{
+   /*******************************************************************************
+    ** In contrast with the DeleteAction, which deletes RECORDS - this is a
+    ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
+    ** e.g., for post-ETL.
+    **
+    ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
+    *******************************************************************************/
+   void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException;
+
+   /*******************************************************************************
+    ** Move a file from a source path, to a destination path.
+    **
+    ** @throws FilesystemException if the move is known to have failed
+    *******************************************************************************/
+   void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException;
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemRecordBackendDetailFields.java

@@ -0,0 +1,32 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.base;
+
+
+/*******************************************************************************
+ ** Define the field names (keys) to be used in the backendDetails structure
+ ** under Records from this (these) modules.
+ *******************************************************************************/
+public interface FilesystemRecordBackendDetailFields
+{
+   String FULL_PATH = "fullPath";
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java

@@ -59,6 +59,10 @@ public abstract class AbstractBaseFilesystemAction<FILE>
     *******************************************************************************/
    public abstract InputStream readFile(FILE file) throws IOException;
 
+   /*******************************************************************************
+    ** Add backend details to records about the file that they are in.
+    *******************************************************************************/
+   protected abstract void addBackendDetailsToRecords(List<QRecord> recordsInFile, FILE file);
 
 
    /*******************************************************************************
@@ -133,6 +137,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
                {
                   String        fileContents  = IOUtils.toString(readFile(file));
                   List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
+                  addBackendDetailsToRecords(recordsInFile, file);
 
                   records.addAll(recordsInFile);
                   break;
@@ -141,6 +146,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
                {
                   String        fileContents  = IOUtils.toString(readFile(file));
                   List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
+                  addBackendDetailsToRecords(recordsInFile, file);
 
                   records.addAll(recordsInFile);
                   break;

src/main/java/com/kingsrook/qqq/backend/module/filesystem/exceptions/FilesystemException.java

@@ -0,0 +1,52 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.exceptions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+
+
+/*******************************************************************************
+ ** Checked exception to be thrown from actions within this module.
+ *******************************************************************************/
+public class FilesystemException extends QException
+{
+   /*******************************************************************************
+    ** Constructor of message
+    **
+    *******************************************************************************/
+   public FilesystemException(String message)
+   {
+      super(message);
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor of message & cause
+    **
+    *******************************************************************************/
+   public FilesystemException(String message, Throwable cause)
+   {
+      super(message, cause);
+   }
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java

@@ -22,26 +22,34 @@
 package com.kingsrook.qqq.backend.module.filesystem.local;
 
 
+import java.io.File;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
 import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
+import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction;
 import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction;
 import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction;
 import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemUpdateAction;
 import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
  ** QQQ Backend module for working with (local) Filesystems.
  *******************************************************************************/
-public class FilesystemBackendModule implements QBackendModuleInterface
+public class FilesystemBackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface
 {
+   private static final Logger LOG = LogManager.getLogger(FilesystemBackendModule.class);
 
 
    /*******************************************************************************
@@ -65,6 +73,7 @@ public class FilesystemBackendModule implements QBackendModuleInterface
    }
 
 
+
    /*******************************************************************************
     ** Method to identify the class used for table-backend details for this module.
     *******************************************************************************/
@@ -117,4 +126,71 @@ public class FilesystemBackendModule implements QBackendModuleInterface
    {
       return (new FilesystemDeleteAction());
    }
+
+
+
+   /*******************************************************************************
+    ** In contrast with the DeleteAction, which deletes RECORDS - this is a
+    ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
+    ** e.g., for post-ETL.
+    **
+    ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
+    *******************************************************************************/
+   @Override
+   public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
+   {
+      File file = new File(fileReference);
+      if(!file.exists())
+      {
+         //////////////////////////////////////////////////////////////////////////////////////////////
+         // if the file doesn't exist, just exit with noop.  don't throw an error - that should only //
+         // happen if the "contract" of the method is broken, and the file still exists              //
+         //////////////////////////////////////////////////////////////////////////////////////////////
+         LOG.debug("Not deleting file [{}], because it does not exist.", file);
+         return;
+      }
+
+      if(!file.delete())
+      {
+         throw (new FilesystemException("Failed to delete file: " + fileReference));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Move a file from a source path, to a destination path.
+    **
+    ** @throws FilesystemException if the delete is known to have failed
+    *******************************************************************************/
+   @Override
+   public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException
+   {
+      File sourceFile        = new File(source);
+      File destinationFile   = new File(destination);
+      File destinationParent = destinationFile.getParentFile();
+
+      if(!sourceFile.exists())
+      {
+         throw (new FilesystemException("Cannot move file " + source + ", as it does not exist."));
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////
+      // if the destination folder doesn't exist, try to make it - and fail if that fails //
+      //////////////////////////////////////////////////////////////////////////////////////
+      if(!destinationParent.exists())
+      {
+         LOG.debug("Making destination directory {} for move", destinationParent.getAbsolutePath());
+         if(!destinationParent.mkdirs())
+         {
+            throw (new FilesystemException("Failed to make destination directory " + destinationParent.getAbsolutePath() + " to move " + source + " into."));
+         }
+      }
+
+      if(!sourceFile.renameTo(destinationFile))
+      {
+         throw (new FilesystemException("Failed to move (rename) file " + source + " to " + destination));
+      }
+   }
+
 }

src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java

@@ -27,9 +27,12 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
 
 
@@ -47,7 +50,14 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
    {
       String fullPath  = getFullPath(table, backendBase);
       File   directory = new File(fullPath);
-      return Arrays.asList(directory.listFiles());
+      File[] files     = directory.listFiles();
+
+      if(files == null)
+      {
+         return Collections.emptyList();
+      }
+
+      return (Arrays.stream(files).filter(File::isFile).toList());
    }
 
 
@@ -61,4 +71,18 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
       return (new FileInputStream(file));
    }
 
+
+
+   /*******************************************************************************
+    ** Add backend details to records about the file that they are in.
+    *******************************************************************************/
+   @Override
+   protected void addBackendDetailsToRecords(List<QRecord> recordsInFile, File file)
+   {
+      recordsInFile.forEach(record ->
+      {
+         record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, file.getAbsolutePath());
+      });
+   }
+
 }

src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java

@@ -0,0 +1,110 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
+
+
+import java.io.File;
+import java.util.Set;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
+import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
+
+
+/*******************************************************************************
+ ** Function body for performing the Cleanup step of a basic ETL process - e.g.,
+ ** after the loading, delete or move the processed file(s).
+ *******************************************************************************/
+public class BasicETLCleanupSourceFilesFunction implements FunctionBody
+{
+   @Override
+   public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
+   {
+      String                  sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
+      QTableMetaData          table           = runFunctionRequest.getInstance().getTable(sourceTableName);
+      QBackendModuleInterface module          = new QBackendModuleDispatcher().getQBackendModule(table.getBackendName());
+      if(!(module instanceof FilesystemBackendModuleInterface filesystemModule))
+      {
+         throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function."));
+      }
+
+      Set<String> sourceFiles = runFunctionRequest.getRecords().stream()
+         .map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))
+         .collect(Collectors.toSet());
+      for(String sourceFile : sourceFiles)
+      {
+         String moveOrDelete = runFunctionRequest.getValueString("moveOrDelete");
+         if("delete".equals(moveOrDelete))
+         {
+            filesystemModule.deleteFile(runFunctionRequest.getInstance(), table, sourceFile);
+         }
+         else if("move".equals(moveOrDelete))
+         {
+            String destinationForMoves = runFunctionRequest.getValueString("destinationForMoves");
+            if(!StringUtils.hasContent(destinationForMoves))
+            {
+               throw (new QException("Field [destinationForMoves] is missing a value."));
+            }
+            File   file            = new File(sourceFile);
+            String destinationPath = destinationForMoves + File.separator + file.getName();
+            filesystemModule.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath);
+         }
+         else
+         {
+            throw (new QException("Unexpected value [" + moveOrDelete + "] for field [moveOrDelete].  Must be either [move] or [delete]."));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QFunctionMetaData defineFunctionMetaData()
+   {
+      return (new QFunctionMetaData()
+         .withName("cleanupSourceFiles")
+         .withCode(new QCodeReference()
+            .withName(this.getClass().getName())
+            .withCodeType(QCodeType.JAVA)
+            .withCodeUsage(QCodeUsage.FUNCTION))
+         .withInputData(new QFunctionInputMetaData()
+            .addField(new QFieldMetaData("moveOrDelete", QFieldType.STRING))
+            .addField(new QFieldMetaData("destinationForMoves", QFieldType.STRING))));
+   }
+}

src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java

@@ -23,25 +23,32 @@ package com.kingsrook.qqq.backend.module.filesystem.s3;
 
 
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
 import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
+import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction;
 import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction;
 import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction;
 import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
+import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
 
 
 /*******************************************************************************
  ** QQQ Backend module for working with AWS S3 filesystems
  *******************************************************************************/
-public class S3BackendModule implements QBackendModuleInterface
+public class S3BackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface
 {
+   private S3Utils s3Utils;
+
 
 
    /*******************************************************************************
@@ -72,7 +79,44 @@ public class S3BackendModule implements QBackendModuleInterface
    @Override
    public Class<? extends QTableBackendDetails> getTableBackendDetailsClass()
    {
-      return S3TableBackendDetails.class;
+      return (S3TableBackendDetails.class);
+   }
+
+
+
+   /*******************************************************************************
+    ** In contrast with the DeleteAction, which deletes RECORDS - this is a
+    ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
+    ** e.g., for post-ETL.
+    **
+    ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
+    *******************************************************************************/
+   @Override
+   public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
+   {
+      QBackendMetaData backend    = instance.getBackend(table.getBackendName());
+      String           bucketName = ((S3BackendMetaData) backend).getBucketName();
+
+      getS3Utils().deleteObject(bucketName, fileReference);
+   }
+
+
+
+
+   /*******************************************************************************
+    ** In contrast with the DeleteAction, which deletes RECORDS - this is a
+    ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
+    ** e.g., for post-ETL.
+    **
+    ** @throws FilesystemException if the move is known to have failed
+    *******************************************************************************/
+   @Override
+   public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException
+   {
+      QBackendMetaData backend    = instance.getBackend(table.getBackendName());
+      String           bucketName = ((S3BackendMetaData) backend).getBucketName();
+
+      getS3Utils().moveObject(bucketName, source, destination);
    }
 
 
@@ -118,4 +162,30 @@ public class S3BackendModule implements QBackendModuleInterface
    {
       return (new S3DeleteAction());
    }
+
+
+
+   /*******************************************************************************
+    ** Setter for s3Utils
+    *******************************************************************************/
+   public void setS3Utils(S3Utils s3Utils)
+   {
+      this.s3Utils = s3Utils;
+   }
+
+
+
+   /*******************************************************************************
+    ** Internal accessor for the s3Utils object - should always use this, not the field.
+    *******************************************************************************/
+   private S3Utils getS3Utils()
+   {
+      if(s3Utils == null)
+      {
+         s3Utils = new S3Utils();
+      }
+
+      return s3Utils;
+   }
+
 }

src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java

@@ -26,8 +26,10 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
@@ -96,4 +98,19 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
    {
       return (getS3Utils().getObjectAsInputStream(s3ObjectSummary));
    }
+
+
+
+   /*******************************************************************************
+    ** Add backend details to records about the file that they are in.
+    *******************************************************************************/
+   @Override
+   protected void addBackendDetailsToRecords(List<QRecord> recordsInFile, S3ObjectSummary s3ObjectSummary)
+   {
+      recordsInFile.forEach(record ->
+      {
+         record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, s3ObjectSummary.getKey());
+      });
+   }
+
 }

src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java

@@ -31,12 +31,17 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.ListObjectsV2Request;
 import com.amazonaws.services.s3.model.ListObjectsV2Result;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
  ** Utility methods for working with AWS S3.
+ **
+ ** Note:  May need a constructor (or similar) in the future that takes the
+ ** S3BackendMetaData - e.g., if we need some metaData to construct the AmazonS3
+ ** (api client) object, such as region, or authentication.
  *******************************************************************************/
 public class S3Utils
 {
@@ -111,8 +116,9 @@ public class S3Utils
    }
 
 
+
    /*******************************************************************************
-    **
+    ** Get the contents (as an InputStream) for an object in s3
     *******************************************************************************/
    public InputStream getObjectAsInputStream(S3ObjectSummary s3ObjectSummary)
    {
@@ -120,6 +126,48 @@ public class S3Utils
    }
 
 
+
+   /*******************************************************************************
+    ** Delete an object (file) from a bucket
+    *******************************************************************************/
+   public void deleteObject(String bucketName, String key) throws FilesystemException
+   {
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // note, aws s3 api does not appear to have any way to check the success or failure here... //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      try
+      {
+         getS3().deleteObject(bucketName, key);
+      }
+      catch(Exception e)
+      {
+         throw (new FilesystemException("Error deleting s3 object " + key + " in bucket " + bucketName, e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Move an object (file) within a bucket
+    *******************************************************************************/
+   public void moveObject(String bucketName, String source, String destination) throws FilesystemException
+   {
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // note, aws s3 api does not appear to have any way to check the success or failure here... //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      try
+      {
+         getS3().copyObject(bucketName, source, bucketName, destination);
+         getS3().deleteObject(bucketName, source);
+      }
+      catch(Exception e)
+      {
+         throw (new FilesystemException("Error moving s3 object " + source + " to " + destination + " in bucket " + bucketName, e));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Setter for AmazonS3 client object.
     *******************************************************************************/
@@ -143,5 +191,4 @@ public class S3Utils
       return s3;
    }
 
-
 }

src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java

@@ -43,8 +43,15 @@ import org.apache.commons.io.FileUtils;
  *******************************************************************************/
 public class TestUtils
 {
+   public static final String TABLE_NAME_PERSON    = "person";
+   public static final String TABLE_NAME_PERSON_S3 = "person-s3";
+
+   ///////////////////////////////////////////////////////////////////
+   // shouldn't be accessed directly, as we append a counter to it. //
+   ///////////////////////////////////////////////////////////////////
+   public static final String BASE_PATH = "/tmp/filesystem-tests";
+
    private static int testInstanceCounter = 0;
-   private final static String BASE_PATH           = "/tmp/filesystem-tests";
 
 
 
@@ -126,7 +133,7 @@ public class TestUtils
    public static QTableMetaData defineLocalFilesystemCSVPersonTable()
    {
       return new QTableMetaData()
-         .withName("person")
+         .withName(TABLE_NAME_PERSON)
          .withLabel("Person")
          .withBackendName(defineLocalFilesystemBackend().getName())
          .withPrimaryKeyField("id")
@@ -165,7 +172,7 @@ public class TestUtils
    public static QTableMetaData defineS3CSVPersonTable()
    {
       return new QTableMetaData()
-         .withName("person-s3")
+         .withName(TABLE_NAME_PERSON_S3)
          .withLabel("Person S3 Table")
          .withBackendName(defineS3Backend().getName())
          .withPrimaryKeyField("id")

src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java

@@ -0,0 +1,174 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.local;
+
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
+import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
+import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+
+/*******************************************************************************
+ ** Unit test for FilesystemBackendModule
+ *******************************************************************************/
+public class FilesystemBackendModuleTest
+{
+   private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist";
+
+
+
+   @BeforeEach
+   public void beforeEach() throws IOException
+   {
+      new FilesystemActionTest().primeFilesystem();
+   }
+
+
+
+   @AfterEach
+   public void afterEach() throws Exception
+   {
+      new FilesystemActionTest().cleanFilesystem();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testDeleteFile() throws Exception
+   {
+      QInstance      qInstance = TestUtils.defineInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
+
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // first list the files - then delete one, then re-list, and assert that we have one fewer //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      List<File> filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+
+      FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
+      filesystemBackendModule.deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath());
+
+      List<File> filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+      Assertions.assertEquals(filesBeforeDelete.size() - 1, filesAfterDelete.size(),
+         "Should be one fewer file listed after deleting one.");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testDeleteFileDoesNotExist() throws Exception
+   {
+      QInstance      qInstance = TestUtils.defineInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
+      // note, we'd like to detect the non-delete, but there's no such info back from aws it appears?                //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      List<File> filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+
+      FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
+      filesystemBackendModule.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST);
+
+      List<File> filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+      Assertions.assertEquals(filesBeforeDelete.size(), filesAfterDelete.size(),
+         "Should be same number of files after deleting bogus path");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testMoveFile() throws Exception
+   {
+      QInstance      qInstance = TestUtils.defineInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
+      String         basePath  = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
+      String         subPath   = basePath + File.separator + "subdir";
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      // first list the files (non-recursively) - then move one into a sub-folder, then re-list, and    //
+      // assert that we have one fewer then list again including sub-folders, and see the changed count //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      List<File> filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+
+      FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
+      String                  originalFilePath        = filesBeforeMove.get(0).getAbsolutePath();
+      String                  movedFilePath           = originalFilePath.replace(basePath, subPath);
+
+      filesystemBackendModule.moveFile(qInstance, table, originalFilePath, movedFilePath);
+
+      List<File> filesAfterMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+      Assertions.assertEquals(filesBeforeMove.size() - 1, filesAfterMove.size(), "Should be one fewer file in the listing after moving one.");
+
+      //////////////////////////////////////////////////////////////////////////
+      // move the file back and assert that the count goes back to the before //
+      //////////////////////////////////////////////////////////////////////////
+      filesystemBackendModule.moveFile(qInstance, table, movedFilePath, originalFilePath);
+
+      List<File> filesAfterMoveBack = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+      Assertions.assertEquals(filesBeforeMove.size(), filesAfterMoveBack.size(), "Should be original number of files after moving back");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testMoveFileDoesNotExit() throws Exception
+   {
+      QInstance      qInstance        = TestUtils.defineInstance();
+      QTableMetaData table            = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
+      String         basePath         = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
+      String         subPath          = basePath + File.separator + "subdir";
+      List<File>     filesBeforeMove  = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
+      String         originalFilePath = filesBeforeMove.get(0).getAbsolutePath();
+      String         movedFilePath    = originalFilePath.replace(basePath, subPath);
+
+      Assertions.assertThrows(FilesystemException.class, () ->
+      {
+         FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
+         filesystemBackendModule.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, movedFilePath);
+      });
+
+   }
+
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java

@@ -43,7 +43,7 @@ public class FilesystemActionTest
    /*******************************************************************************
     ** Set up the file system
     *******************************************************************************/
-   protected void primeFilesystem() throws IOException
+   public void primeFilesystem() throws IOException
    {
       TestUtils.cleanInstanceFiles();
       TestUtils.increaseTestInstanceCounter();
@@ -94,7 +94,7 @@ public class FilesystemActionTest
    /*******************************************************************************
     **
     *******************************************************************************/
-   protected void cleanFilesystem() throws IOException
+   public void cleanFilesystem() throws IOException
    {
       TestUtils.cleanInstanceFiles();
    }

src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java

@@ -27,6 +27,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
 import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
 import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
@@ -70,6 +71,9 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
       QueryRequest queryRequest = initQueryRequest();
       QueryResult  queryResult  = new FilesystemQueryAction().execute(queryRequest);
       Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows");
+      Assertions.assertTrue(queryResult.getRecords().stream()
+            .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
+         "All records should have a full-path in their backend details, matching the test folder name");
    }
 
 

src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java

@@ -35,7 +35,7 @@ import org.junit.jupiter.api.extension.ExtendWith;
 
 
 /*******************************************************************************
- **
+ ** Base class for tests that want to be able to work with localstack s3.
  *******************************************************************************/
 @ExtendWith(LocalstackDockerExtension.class)
 @LocalstackDockerProperties(services = { ServiceName.S3 }, portEdge = "2960", portElasticSearch = "2961")

src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java

@@ -0,0 +1,151 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3;
+
+
+import java.util.List;
+import java.util.UUID;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+
+/*******************************************************************************
+ ** Unit test for S3BackendModule
+ *******************************************************************************/
+public class S3BackendModuleTest extends BaseS3Test
+{
+   private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testDeleteFile() throws Exception
+   {
+      QInstance      qInstance = TestUtils.defineInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
+
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // first list the files - then delete one, then re-list, and assert that we have one fewer //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      List<S3ObjectSummary> s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
+
+      S3BackendModule s3BackendModule = new S3BackendModule();
+      s3BackendModule.setS3Utils(getS3Utils());
+      s3BackendModule.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey());
+
+      List<S3ObjectSummary> s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
+      Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(),
+         "Should be one fewer file listed after deleting one.");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testDeleteFileDoesNotExist() throws Exception
+   {
+      QInstance      qInstance = TestUtils.defineInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
+      // note, we'd like to detect the non-delete, but there's no such info back from aws it appears?                //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      List<S3ObjectSummary> s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
+
+      S3BackendModule s3BackendModule = new S3BackendModule();
+      s3BackendModule.setS3Utils(getS3Utils());
+      s3BackendModule.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST);
+
+      List<S3ObjectSummary> s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
+      Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size(), s3ObjectSummariesAfterDelete.size(),
+         "Should be same number of files after deleting bogus path");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testMoveFile() throws Exception
+   {
+      QInstance      qInstance = TestUtils.defineInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
+      String         subPath   = TEST_FOLDER + "/" + SUB_FOLDER;
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      // first list the files (non-recursively) - then move one into a sub-folder, then re-list, and    //
+      // assert that we have one fewer then list again including sub-folders, and see the changed count //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      List<S3ObjectSummary> s3ObjectSummariesBeforeMove            = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
+      List<S3ObjectSummary> s3ObjectSummariesInSubFolderBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, subPath, false);
+      List<S3ObjectSummary> s3ObjectSummariesRecursiveBeforeMove   = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, true);
+
+      S3BackendModule s3BackendModule = new S3BackendModule();
+      s3BackendModule.setS3Utils(getS3Utils());
+      String key = s3ObjectSummariesBeforeMove.get(0).getKey();
+      s3BackendModule.moveFile(qInstance, table, key, key.replaceFirst(TEST_FOLDER, subPath));
+
+      List<S3ObjectSummary> s3ObjectSummariesAfterMove            = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
+      List<S3ObjectSummary> s3ObjectSummariesRecursiveAfterMove   = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, true);
+      List<S3ObjectSummary> s3ObjectSummariesInSubFolderAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, subPath, false);
+
+      Assertions.assertEquals(s3ObjectSummariesBeforeMove.size() - 1, s3ObjectSummariesAfterMove.size(),
+         "Should be one fewer file in the non-recursive listing after moving one.");
+      Assertions.assertEquals(s3ObjectSummariesRecursiveBeforeMove.size(), s3ObjectSummariesRecursiveAfterMove.size(),
+         "Should be same number of files in the recursive listing before and after the move");
+      Assertions.assertEquals(s3ObjectSummariesInSubFolderBeforeMove.size() + 1, s3ObjectSummariesInSubFolderAfterMove.size(),
+         "Should be one move file in the sub-folder listing after moving one.");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testMoveFileDoesNotExit() throws Exception
+   {
+      QInstance      qInstance = TestUtils.defineInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
+      String         subPath   = TEST_FOLDER + "/" + SUB_FOLDER;
+
+      S3BackendModule s3BackendModule = new S3BackendModule();
+      s3BackendModule.setS3Utils(getS3Utils());
+
+      Assertions.assertThrows(FilesystemException.class, () ->
+         s3BackendModule.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, subPath + "/" + UUID.randomUUID())
+      );
+   }
+
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java

@@ -27,6 +27,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
 import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
 import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -49,6 +50,9 @@ public class S3QueryActionTest extends BaseS3Test
       s3QueryAction.setS3Utils(getS3Utils());
       QueryResult queryResult = s3QueryAction.execute(queryRequest);
       Assertions.assertEquals(5, queryResult.getRecords().size(), "Expected # of rows from unfiltered query");
+      Assertions.assertTrue(queryResult.getRecords().stream()
+            .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseS3Test.TEST_FOLDER)),
+         "All records should have a full-path in their backend details, matching the test folder name");
    }