ES-52: updated UpdateAction to act same as insert or gets for 'retryable server errors'

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java

@@ -1178,7 +1178,7 @@ public class BaseAPIActionUtil
     *******************************************************************************/
    protected boolean shouldBeRetryableServerErrorException(QHttpResponse qResponse)
    {
-      if(actionInput instanceof QueryInput || actionInput instanceof GetInput)
+      if(actionInput instanceof QueryInput || actionInput instanceof GetInput || actionInput instanceof UpdateInput)
       {
          return (qResponse.getStatusCode() != null && qResponse.getStatusCode() >= 500);
       }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java

@@ -36,7 +36,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -69,17 +68,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
    /*******************************************************************************
     ** List the files for a table - to be implemented in module-specific subclasses.
     *******************************************************************************/
-   public List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase) throws QException
+   public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase);
-   {
-      return (listFiles(table, backendBase, null));
-   }
-
-
-
-   /*******************************************************************************
-    ** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses.
-    *******************************************************************************/
-   public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException;
 
    /*******************************************************************************
     ** Read the contents of a file - to be implemented in module-specific subclasses.
@@ -192,7 +181,6 @@ public abstract class AbstractBaseFilesystemAction<FILE>
    /*******************************************************************************
     ** Generic implementation of the execute method from the QueryInterface
     *******************************************************************************/
-   @SuppressWarnings("checkstyle:Indentation")
    public QueryOutput executeQuery(QueryInput queryInput) throws QException
    {
       preAction(queryInput.getBackend());
@@ -203,76 +191,51 @@ public abstract class AbstractBaseFilesystemAction<FILE>
 
          QTableMetaData                        table        = queryInput.getTable();
          AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
-         List<FILE>                            files        = listFiles(table, queryInput.getBackend(), queryInput.getFilter());
+         List<FILE>                            files        = listFiles(table, queryInput.getBackend());
 
          for(FILE file : files)
          {
             LOG.info("Processing file: " + getFullPathForFile(file));
-
+            switch(tableDetails.getRecordFormat())
-            InputStream inputStream = readFile(file);
-            switch(tableDetails.getCardinality())
             {
-               case MANY:
+               case CSV:
                {
-                  switch(tableDetails.getRecordFormat())
+                  String fileContents = IOUtils.toString(readFile(file));
+                  fileContents = customizeFileContentsAfterReading(table, fileContents);
+
+                  if(queryInput.getRecordPipe() != null)
                   {
-                     case CSV:
+                     new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record ->
                      {
-                        String fileContents = IOUtils.toString(inputStream);
+                        ////////////////////////////////////////////////////////////////////////////////////////////
-                        fileContents = customizeFileContentsAfterReading(table, fileContents);
+                        // Before the records go into the pipe, make sure their backend details are added to them //
-
+                        ////////////////////////////////////////////////////////////////////////////////////////////
-                        if(queryInput.getRecordPipe() != null)
+                        addBackendDetailsToRecord(record, file);
-                        {
+                     }));
-                           new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record ->
+                  }
-                           {
+                  else
-                              ////////////////////////////////////////////////////////////////////////////////////////////
+                  {
-                              // Before the records go into the pipe, make sure their backend details are added to them //
+                     List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
-                              ////////////////////////////////////////////////////////////////////////////////////////////
+                     addBackendDetailsToRecords(recordsInFile, file);
-                              addBackendDetailsToRecord(record, file);
+                     queryOutput.addRecords(recordsInFile);
-                           }));
-                        }
-                        else
-                        {
-                           List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
-                           addBackendDetailsToRecords(recordsInFile, file);
-                           queryOutput.addRecords(recordsInFile);
-                        }
-                        break;
-                     }
-                     case JSON:
-                     {
-                        String fileContents = IOUtils.toString(inputStream);
-                        fileContents = customizeFileContentsAfterReading(table, fileContents);
-
-                        // todo - pipe support!!
-                        List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
-                        addBackendDetailsToRecords(recordsInFile, file);
-
-                        queryOutput.addRecords(recordsInFile);
-                        break;
-                     }
-                     default:
-                     {
-                        throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat());
-                     }
                   }
                   break;
                }
-               case ONE:
+               case JSON:
                {
-                  String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table);
+                  String fileContents = IOUtils.toString(readFile(file));
+                  fileContents = customizeFileContentsAfterReading(table, fileContents);
 
-                  byte[] bytes = inputStream.readAllBytes();
+                  // todo - pipe support!!
-                  QRecord record = new QRecord()
+                  List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
-                     .withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase)
+                  addBackendDetailsToRecords(recordsInFile, file);
-                     .withValue(tableDetails.getContentsFieldName(), bytes);
-                  queryOutput.addRecord(record);
 
+                  queryOutput.addRecords(recordsInFile);
                   break;
                }
                default:
                {
-                  throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality());
+                  throw new NotImplementedException("Filesystem record format " + tableDetails.getRecordFormat() + " is not yet implemented");
                }
             }
          }
@@ -379,8 +342,8 @@ public abstract class AbstractBaseFilesystemAction<FILE>
          {
             for(QRecord record : insertInput.getRecords())
             {
-               String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
+               String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString("fileName"));
-               writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
+               writeFile(backend, fullPath, record.getValueByteArray("contents"));
                record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
                output.addRecord(record);
             }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java

@@ -35,11 +35,6 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
    private RecordFormat recordFormat;
    private Cardinality  cardinality;
 
-   ///////////////////////////////////////////////////////////////////////////////////////////////////
-   // todo default these to null, and give validation error if not set for a cardinality=ONE table? //
-   ///////////////////////////////////////////////////////////////////////////////////////////////////
-   private String contentsFieldName = "contents";
-   private String fileNameFieldName = "fileName";
 
 
    /*******************************************************************************
@@ -180,67 +175,4 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
       return ((T) this);
    }
 
-
-
-   /*******************************************************************************
-    ** Getter for contentsFieldName
-    *******************************************************************************/
-   public String getContentsFieldName()
-   {
-      return (this.contentsFieldName);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for contentsFieldName
-    *******************************************************************************/
-   public void setContentsFieldName(String contentsFieldName)
-   {
-      this.contentsFieldName = contentsFieldName;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for contentsFieldName
-    *******************************************************************************/
-   public AbstractFilesystemTableBackendDetails withContentsFieldName(String contentsFieldName)
-   {
-      this.contentsFieldName = contentsFieldName;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for fileNameFieldName
-    *******************************************************************************/
-   public String getFileNameFieldName()
-   {
-      return (this.fileNameFieldName);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for fileNameFieldName
-    *******************************************************************************/
-   public void setFileNameFieldName(String fileNameFieldName)
-   {
-      this.fileNameFieldName = fileNameFieldName;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for fileNameFieldName
-    *******************************************************************************/
-   public AbstractFilesystemTableBackendDetails withFileNameFieldName(String fileNameFieldName)
-   {
-      this.fileNameFieldName = fileNameFieldName;
-      return (this);
-   }
-
-
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java

@@ -23,36 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
 
 
 import java.io.File;
-import java.io.FileFilter;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
-import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
-import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
-import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOCase;
-import org.apache.commons.io.filefilter.AndFileFilter;
-import org.apache.commons.io.filefilter.NameFileFilter;
-import org.apache.commons.io.filefilter.OrFileFilter;
-import org.apache.commons.io.filefilter.TrueFileFilter;
-import org.apache.commons.io.filefilter.WildcardFileFilter;
 
 
 /*******************************************************************************
@@ -68,66 +51,12 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
     ** List the files for this table.
     *******************************************************************************/
    @Override
-   public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
+   public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase)
    {
+      // todo - needs rewritten to do globbing...
       String fullPath  = getFullBasePath(table, backendBase);
       File   directory = new File(fullPath);
-      File[] files     = null;
+      File[] files     = directory.listFiles();
-
-      AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
-
-      FileFilter fileFilter = TrueFileFilter.INSTANCE;
-
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // if each file is its own record (ONE), then we may need to do filtering of the directory listing based on the input filter //
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      if(Cardinality.ONE.equals(tableBackendDetails.getCardinality()))
-      {
-         if(filter != null && filter.hasAnyCriteria())
-         {
-            List<FileFilter> fileFilterList = new ArrayList<>();
-            for(QFilterCriteria criteria : filter.getCriteria())
-            {
-               if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
-               {
-                  if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
-                  {
-                     fileFilterList.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(0))));
-                  }
-                  else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
-                  {
-                     List<NameFileFilter> nameInFilters = new ArrayList<>();
-                     for(int i = 0; i < criteria.getValues().size(); i++)
-                     {
-                        nameInFilters.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(i))));
-                     }
-                     fileFilterList.add(new OrFileFilter(nameInFilters));
-                  }
-                  else
-                  {
-                     throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
-                  }
-               }
-               else
-               {
-                  throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
-               }
-            }
-
-            fileFilter = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? new AndFileFilter(fileFilterList) : new OrFileFilter(fileFilterList);
-         }
-      }
-
-      ///////////////////////////////////////////////////////////////////////////////////////////
-      // if the table has a glob specified, add it as an AND to the filter built to this point //
-      ///////////////////////////////////////////////////////////////////////////////////////////
-      if(StringUtils.hasContent(tableBackendDetails.getGlob()))
-      {
-         WildcardFileFilter globFilenameFilter = new WildcardFileFilter(tableBackendDetails.getGlob(), IOCase.INSENSITIVE);
-         fileFilter = new AndFileFilter(List.of(globFilenameFilter, fileFilter));
-      }
-
-      files = directory.listFiles(fileFilter);
 
       if(files == null)
       {

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java

@@ -59,45 +59,31 @@ public class FilesystemSyncStep implements BackendStep
     *******************************************************************************/
    @Override
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
-   {
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // defer to a private method here, so we can add a type-parameter for that method to use              //
-      // would think we could do that here, but get compiler error, since this method comes from base class //
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////
-      doRun(runBackendStepInput, runBackendStepOutput);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private <F> void doRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
       QTableMetaData sourceTable     = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
       QTableMetaData archiveTable    = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
       QTableMetaData processingTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
 
-      QBackendMetaData                    sourceBackend    = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName());
+      QBackendMetaData                 sourceBackend    = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName());
-      FilesystemBackendModuleInterface<F> sourceModule     = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
+      FilesystemBackendModuleInterface sourceModule     = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
-      AbstractBaseFilesystemAction<F>     sourceActionBase = sourceModule.getActionBase();
+      AbstractBaseFilesystemAction     sourceActionBase = sourceModule.getActionBase();
       sourceActionBase.preAction(sourceBackend);
-      Map<String, F> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
+      Map<String, Object> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
 
-      QBackendMetaData                    archiveBackend    = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName());
+      QBackendMetaData                 archiveBackend    = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName());
-      FilesystemBackendModuleInterface<F> archiveModule     = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
+      FilesystemBackendModuleInterface archiveModule     = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
-      AbstractBaseFilesystemAction<F>     archiveActionBase = archiveModule.getActionBase();
+      AbstractBaseFilesystemAction     archiveActionBase = archiveModule.getActionBase();
       archiveActionBase.preAction(archiveBackend);
       Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet();
 
-      QBackendMetaData                    processingBackend    = runBackendStepInput.getInstance().getBackendForTable(processingTable.getName());
+      QBackendMetaData                 processingBackend    = runBackendStepInput.getInstance().getBackendForTable(processingTable.getName());
-      FilesystemBackendModuleInterface<F> processingModule     = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
+      FilesystemBackendModuleInterface processingModule     = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
-      AbstractBaseFilesystemAction<F>     processingActionBase = processingModule.getActionBase();
+      AbstractBaseFilesystemAction     processingActionBase = processingModule.getActionBase();
       processingActionBase.preAction(processingBackend);
 
       Integer maxFilesToSync  = runBackendStepInput.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
       int     syncedFileCount = 0;
-      for(Map.Entry<String, F> sourceEntry : sourceFiles.entrySet())
+      for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet())
       {
          try
          {
@@ -105,22 +91,20 @@ public class FilesystemSyncStep implements BackendStep
             if(!archiveFiles.contains(sourceFileName))
             {
                LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]");
-               try(InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue()))
+               InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue());
+               byte[]      bytes       = inputStream.readAllBytes();
+
+               String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
+               archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
+
+               String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
+               processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
+               syncedFileCount++;
+
+               if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)
                {
-                  byte[] bytes = inputStream.readAllBytes();
+                  LOG.info("Breaking after syncing " + syncedFileCount + " files");
-
+                  break;
-                  String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
-                  archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
-
-                  String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
-                  processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
-                  syncedFileCount++;
-
-                  if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)
-                  {
-                     LOG.info("Breaking after syncing " + syncedFileCount + " files");
-                     break;
-                  }
                }
             }
          }
@@ -136,12 +120,12 @@ public class FilesystemSyncStep implements BackendStep
    /*******************************************************************************
     **
     *******************************************************************************/
-   private <F> Map<String, F> getFileNames(AbstractBaseFilesystemAction<F> actionBase, QTableMetaData table, QBackendMetaData backend) throws QException
+   private Map<String, Object> getFileNames(AbstractBaseFilesystemAction actionBase, QTableMetaData table, QBackendMetaData backend)
    {
-      List<F>        files = actionBase.listFiles(table, backend);
+      List<Object>        files = actionBase.listFiles(table, backend);
-      Map<String, F> rs    = new LinkedHashMap<>();
+      Map<String, Object> rs    = new LinkedHashMap<>();
 
-      for(F file : files)
+      for(Object file : files)
       {
          String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table);
          rs.put(fileName, file);

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java

@@ -30,9 +30,7 @@ import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -128,7 +126,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
     ** List the files for a table.
     *******************************************************************************/
    @Override
-   public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
+   public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase)
    {
       S3BackendMetaData                     s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase);
       AbstractFilesystemTableBackendDetails tableDetails      = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
@@ -140,7 +138,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
       ////////////////////////////////////////////////////////////////////
       // todo - look at metadata to configure the s3 client here?       //
       ////////////////////////////////////////////////////////////////////
-      return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob, filter, tableDetails);
+      return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob);
    }
 
 

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java

@@ -37,14 +37,7 @@ import com.amazonaws.services.s3.model.ListObjectsV2Request;
 import com.amazonaws.services.s3.model.ListObjectsV2Result;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
-import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
 
@@ -68,19 +61,7 @@ public class S3Utils
     ** List the objects in an S3 bucket matching a glob, per:
     ** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
     *******************************************************************************/
-   public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob) throws QException
+   public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob)
-   {
-      return listObjectsInBucketMatchingGlob(bucketName, path, glob, null, null);
-   }
-
-
-
-   /*******************************************************************************
-    ** List the objects in an S3 bucket matching a glob, per:
-    ** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
-    ** and also - (possibly) apply a file-name filter (based on the table's details).
-    *******************************************************************************/
-   public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
    {
       //////////////////////////////////////////////////////////////////////////////////////////////////
       // s3 list requests find nothing if the path starts with a /, so strip away any leading slashes //
@@ -96,28 +77,6 @@ public class S3Utils
          prefix = prefix.substring(0, prefix.indexOf('*'));
       }
 
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////
-      // for a file-per-record (ONE) table, we may need to apply the filter to listing.                    //
-      // but for MANY tables, the filtering would be done on the records after they came out of the files. //
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////
-      boolean useQQueryFilter = false;
-      if(tableDetails != null && Cardinality.ONE.equals(tableDetails.getCardinality()))
-      {
-         useQQueryFilter = true;
-      }
-
-      if(filter != null && useQQueryFilter)
-      {
-         if(filter.getCriteria() != null && filter.getCriteria().size() == 1)
-         {
-            QFilterCriteria criteria = filter.getCriteria().get(0);
-            if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS))
-            {
-               prefix += "/" + criteria.getValues().get(0);
-            }
-         }
-      }
-
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob.                //
       // That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. //
@@ -127,7 +86,6 @@ public class S3Utils
       {
          glob = "";
       }
-
       if(!glob.contains("*"))
       {
          if(glob.equals(""))
@@ -156,7 +114,7 @@ public class S3Utils
          {
             listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken());
          }
-         LOG.info("Listing bucket=" + bucketName + ", path=" + path + ", prefix=" + prefix + ", glob=" + glob);
+         LOG.info("Listing bucket=" + bucketName + ", path=" + path);
          listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request);
 
          //////////////////////////////////
@@ -191,14 +149,6 @@ public class S3Utils
                continue;
             }
 
-            ///////////////////////////////////////////////////////////////////////////////////
-            // if we're a file-per-record table, and we have a filter, compare the key to it //
-            ///////////////////////////////////////////////////////////////////////////////////
-            if(!doesObjectKeyMatchFilter(key, filter, tableDetails))
-            {
-               continue;
-            }
-
             rs.add(objectSummary);
          }
       }
@@ -209,95 +159,6 @@ public class S3Utils
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private boolean doesObjectKeyMatchFilter(String key, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
-   {
-      if(filter == null || !filter.hasAnyCriteria())
-      {
-         return (true);
-      }
-
-      Path path = Path.of(URI.create("file:///" + key));
-
-      ////////////////////////////////////////////////////////////////////////////////////////////////////
-      // foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches //
-      ////////////////////////////////////////////////////////////////////////////////////////////////////
-      for(QFilterCriteria criteria : filter.getCriteria())
-      {
-         boolean matches = doesObjectKeyMatchOneCriteria(criteria, tableDetails, path);
-
-         if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
-         {
-            ////////////////////////////////////////////////////////////////////////////////
-            // if it's not a match, and it's an AND filter, then the whole thing is false //
-            ////////////////////////////////////////////////////////////////////////////////
-            return (false);
-         }
-
-         if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
-         {
-            ////////////////////////////////////////////////////////////
-            // if it's an OR filter, and we've a match, return a true //
-            ////////////////////////////////////////////////////////////
-            return (true);
-         }
-      }
-
-      //////////////////////////////////////////////////////////////////////
-      // if we didn't return above, return now                            //
-      // for an OR - if we didn't find something true, then return false. //
-      // else, an AND - if we didn't find a false, we can return true.    //
-      //////////////////////////////////////////////////////////////////////
-      if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
-      {
-         return (false);
-      }
-
-      return (true);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static boolean doesObjectKeyMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException
-   {
-      if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
-      {
-         if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
-         {
-            return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path));
-         }
-         else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
-         {
-            boolean anyMatch = false;
-            for(int i = 0; i < criteria.getValues().size(); i++)
-            {
-               if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path))
-               {
-                  anyMatch = true;
-                  break;
-               }
-            }
-
-            return (anyMatch);
-         }
-         else
-         {
-            throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
-         }
-      }
-      else
-      {
-         throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
-      }
-   }
-
-
-
    /*******************************************************************************
     ** Get the contents (as an InputStream) for an object in s3
     *******************************************************************************/
@@ -384,5 +245,4 @@ public class S3Utils
 
       return amazonS3;
    }
-
 }

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java

@@ -25,11 +25,6 @@ package com.kingsrook.qqq.backend.module.filesystem.local;
 import java.io.File;
 import java.io.IOException;
 import java.util.List;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@@ -41,8 +36,6 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
 /*******************************************************************************
@@ -54,9 +47,6 @@ public class FilesystemBackendModuleTest
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
    @BeforeEach
    public void beforeEach() throws IOException
    {
@@ -65,9 +55,6 @@ public class FilesystemBackendModuleTest
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
    @AfterEach
    public void afterEach() throws Exception
    {
@@ -76,78 +63,6 @@ public class FilesystemBackendModuleTest
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void testListFiles() throws QException
-   {
-      QInstance      qInstance = TestUtils.defineInstance();
-      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
-
-      AbstractFilesystemAction abstractFilesystemAction = new AbstractFilesystemAction();
-      QBackendMetaData         backend                  = qInstance.getBackendForTable(table.getName());
-
-      //////////////////////////////////////////////////////////
-      // with no filter given, all (3) files should come back //
-      //////////////////////////////////////////////////////////
-      List<File> files = abstractFilesystemAction.listFiles(table, backend);
-      assertEquals(3, files.size());
-
-      /////////////////////////////////////////
-      // filter for a file name that's found //
-      /////////////////////////////////////////
-      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
-      assertEquals(1, files.size());
-      assertEquals("BLOB-2.txt", files.get(0).getName());
-
-      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
-      assertEquals(1, files.size());
-      assertEquals("BLOB-1.txt", files.get(0).getName());
-
-      ///////////////////////////////////
-      // filter for 2 names that exist //
-      ///////////////////////////////////
-      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
-      assertEquals(2, files.size());
-
-      /////////////////////////////////////////////
-      // filter for a file name that isn't found //
-      /////////////////////////////////////////////
-      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
-      assertEquals(0, files.size());
-
-      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
-      assertEquals(1, files.size());
-
-      ////////////////////////////////////////////////////
-      // 2 criteria, and'ed, and can't match, so find 0 //
-      ////////////////////////////////////////////////////
-      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
-      assertEquals(0, files.size());
-
-      //////////////////////////////////////////////////
-      // 2 criteria, or'ed, and both match, so find 2 //
-      //////////////////////////////////////////////////
-      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
-         .withBooleanOperator(QQueryFilter.BooleanOperator.OR));
-      assertEquals(2, files.size());
-
-      //////////////////////////////////////
-      // ensure unsupported filters throw //
-      //////////////////////////////////////
-      assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
-         .hasMessageContaining("Unable to query filesystem table by field");
-      assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
-         .hasMessageContaining("Unable to query filename field using operator");
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java

@@ -89,7 +89,6 @@ public class FilesystemActionTest extends BaseTest
 
       writePersonJSONFiles(baseDirectory);
       writePersonCSVFiles(baseDirectory);
-      writeBlobFiles(baseDirectory);
    }
 
 
@@ -159,37 +158,6 @@ public class FilesystemActionTest extends BaseTest
 
 
 
-   /*******************************************************************************
-    ** Write some data files into the directory for the filesystem module.
-    *******************************************************************************/
-   private void writeBlobFiles(File baseDirectory) throws IOException
-   {
-      String fullPath = baseDirectory.getAbsolutePath();
-      if(TestUtils.defineLocalFilesystemBlobTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
-      {
-         if(StringUtils.hasContent(details.getBasePath()))
-         {
-            fullPath += File.separatorChar + details.getBasePath();
-         }
-      }
-      fullPath += File.separatorChar;
-
-      String data1 = """
-         Hello, Blob
-         """;
-      FileUtils.writeStringToFile(new File(fullPath + "BLOB-1.txt"), data1);
-
-      String data2 = """
-         Hi Bob""";
-      FileUtils.writeStringToFile(new File(fullPath + "BLOB-2.txt"), data2);
-
-      String data3 = """
-         # Hi MD...""";
-      FileUtils.writeStringToFile(new File(fullPath + "BLOB-3.md"), data3);
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java

@@ -23,9 +23,6 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
 
 
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -35,10 +32,8 @@ import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractPostReadFileCustomizer;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.FilesystemTableCustomizers;
-import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
 /*******************************************************************************
@@ -56,8 +51,8 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
       QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
-      assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
+      Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
-      assertTrue(queryOutput.getRecords().stream()
+      Assertions.assertTrue(queryOutput.getRecords().stream()
             .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
          "All records should have a full-path in their backend details, matching the test folder name");
    }
@@ -79,48 +74,14 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
 
       queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
       QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
-      assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
+      Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
-      assertTrue(
+      Assertions.assertTrue(
          queryOutput.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")),
          "All records should have their email addresses up-shifted.");
    }
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   public void testQueryForCardinalityOne() throws QException
-   {
-      QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
-      queryInput.setFilter(new QQueryFilter());
-      QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
-      assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
-
-      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
-      queryOutput = new FilesystemQueryAction().execute(queryInput);
-      assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
-      assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
-
-      ////////////////////////////////////////////////////////////////
-      // put a glob on the table - now should only find 2 txt files //
-      ////////////////////////////////////////////////////////////////
-      QInstance instance = TestUtils.defineInstance();
-      ((FilesystemTableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).getBackendDetails()))
-         .withGlob("*.txt");
-      reInitInstanceInContext(instance);
-
-      queryInput.setFilter(new QQueryFilter());
-      queryOutput = new FilesystemQueryAction().execute(queryInput);
-      assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
    public static class ValueUpshifter extends AbstractPostReadFileCustomizer
    {
       @Override

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java

@@ -26,7 +26,7 @@ import java.util.List;
 import java.util.stream.Collectors;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@@ -187,7 +187,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QException
+   private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException
    {
       S3BackendModule  module     = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
       AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
@@ -207,7 +207,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QException
+   private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException
    {
       S3BackendModule  module     = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
       AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java

@@ -62,9 +62,6 @@ public class BaseS3Test extends BaseTest
       amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2());
       amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test");
       amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
-      amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-1.txt", "Hello, Blob");
-      amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-2.txt", "Hi, Bob");
-      amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-3.md", "# Hi, MD");
    }
 
 

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java

@@ -25,11 +25,6 @@ package com.kingsrook.qqq.backend.module.filesystem.s3;
 import java.util.List;
 import java.util.UUID;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@@ -37,9 +32,6 @@ import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemExceptio
 import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
 /*******************************************************************************
@@ -51,83 +43,6 @@ public class S3BackendModuleTest extends BaseS3Test
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void testListFiles() throws QException
-   {
-      QInstance        qInstance = TestUtils.defineInstance();
-      QTableMetaData   table     = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_S3);
-      QBackendMetaData backend   = qInstance.getBackendForTable(table.getName());
-
-      //////////////////////////////////////////////////////
-      // set up the backend module (e.g., for localstack) //
-      //////////////////////////////////////////////////////
-      S3BackendModule  s3BackendModule = new S3BackendModule();
-      AbstractS3Action actionBase      = (AbstractS3Action) s3BackendModule.getActionBase();
-      actionBase.setS3Utils(getS3Utils());
-
-      //////////////////////////////////////////////////////////
-      // with no filter given, all (3) files should come back //
-      //////////////////////////////////////////////////////////
-      List<S3ObjectSummary> files = actionBase.listFiles(table, backend);
-      assertEquals(3, files.size());
-
-      /////////////////////////////////////////
-      // filter for a file name that's found //
-      /////////////////////////////////////////
-      files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
-      assertEquals(1, files.size());
-      assertThat(files.get(0).getKey()).contains("BLOB-2.txt");
-
-      files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
-      assertEquals(1, files.size());
-      assertThat(files.get(0).getKey()).contains("BLOB-1.txt");
-
-      ///////////////////////////////////
-      // filter for 2 names that exist //
-      ///////////////////////////////////
-      files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
-      assertEquals(2, files.size());
-
-      /////////////////////////////////////////////
-      // filter for a file name that isn't found //
-      /////////////////////////////////////////////
-      files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
-      assertEquals(0, files.size());
-
-      files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
-      assertEquals(1, files.size());
-
-      ////////////////////////////////////////////////////
-      // 2 criteria, and'ed, and can't match, so find 0 //
-      ////////////////////////////////////////////////////
-      files = actionBase.listFiles(table, backend, new QQueryFilter(
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
-      assertEquals(0, files.size());
-
-      //////////////////////////////////////////////////
-      // 2 criteria, or'ed, and both match, so find 2 //
-      //////////////////////////////////////////////////
-      files = actionBase.listFiles(table, backend, new QQueryFilter(
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
-         .withBooleanOperator(QQueryFilter.BooleanOperator.OR));
-      assertEquals(2, files.size());
-
-      //////////////////////////////////////
-      // ensure unsupported filters throw //
-      //////////////////////////////////////
-      assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
-         .hasMessageContaining("Unable to query filesystem table by field");
-      assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
-         .hasMessageContaining("Unable to query filename field using operator");
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java

@@ -23,19 +23,13 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
 
 
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
-import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
-import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
 /*******************************************************************************
@@ -72,39 +66,4 @@ public class S3QueryActionTest extends BaseS3Test
       return queryInput;
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   public void testQueryForCardinalityOne() throws QException
-   {
-      QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_S3);
-      queryInput.setFilter(new QQueryFilter());
-
-      S3QueryAction s3QueryAction = new S3QueryAction();
-      s3QueryAction.setS3Utils(getS3Utils());
-
-      QueryOutput queryOutput = s3QueryAction.execute(queryInput);
-      assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
-
-      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
-      queryOutput = s3QueryAction.execute(queryInput);
-      assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
-      assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
-
-      ////////////////////////////////////////////////////////////////
-      // put a glob on the table - now should only find 2 txt files //
-      ////////////////////////////////////////////////////////////////
-      QInstance instance = TestUtils.defineInstance();
-      ((S3TableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_S3).getBackendDetails()))
-         .withGlob("*.txt");
-      reInitInstanceInContext(instance);
-
-      queryInput.setFilter(new QQueryFilter());
-      queryOutput = s3QueryAction.execute(queryInput);
-      assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
-   }
-
 }

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java

@@ -22,10 +22,10 @@
 package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
 
 
+import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
 import org.apache.commons.io.IOUtils;
 import org.junit.jupiter.api.Test;
@@ -42,14 +42,14 @@ public class S3UtilsTest extends BaseS3Test
     **
     *******************************************************************************/
    @Test
-   public void testListObjectsInBucketAtPath() throws QException
+   public void testListObjectsInBucketAtPath()
    {
       S3Utils s3Utils = getS3Utils();
       assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders");
       assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders");
       assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders");
       assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders");
-      assertEquals(7, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
+      assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
       assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash");
       assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash");
       assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes");
@@ -60,8 +60,8 @@ public class S3UtilsTest extends BaseS3Test
       assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /");
       assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s");
       assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string");
-      assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
+      assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
-      assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
+      assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
    }
 
 
@@ -70,7 +70,7 @@ public class S3UtilsTest extends BaseS3Test
     **
     *******************************************************************************/
    @Test
-   public void testGetObjectAsInputStream() throws Exception
+   public void testGetObjectAsInputStream() throws IOException
    {
       S3Utils               s3Utils           = getS3Utils();
       List<S3ObjectSummary> s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", "");