Simplify file listing by replacing filters with requested paths

Refactor file listing mechanisms to replace the use of complex query filters with simpler, path-based requests. Updated module-specific implementations and removed unused filtering logic. Updated tests (zombie'ing some)

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java

@@ -57,8 +57,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSett
 import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil;
 import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
@@ -111,9 +111,10 @@ public abstract class AbstractBaseFilesystemAction<FILE>
    public abstract Instant getFileModifyDate(FILE file);
 
    /*******************************************************************************
-    ** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses.
+    ** List the files for a table - or optionally, just a single file name -
+    ** to be implemented in module-specific subclasses.
     *******************************************************************************/
-   public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException;
+   public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedSingleFileName) throws QException;
 
    /*******************************************************************************
     ** Read the contents of a file - to be implemented in module-specific subclasses.
@@ -278,11 +279,26 @@ public abstract class AbstractBaseFilesystemAction<FILE>
 
       try
       {
-         QueryOutput queryOutput = new QueryOutput(queryInput);
-
          QTableMetaData                        table        = queryInput.getTable();
          AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
-         List<FILE>                            files        = listFiles(table, queryInput.getBackend(), queryInput.getFilter());
+
+         QueryOutput queryOutput = new QueryOutput(queryInput);
+
+         String       requestedPath = null;
+         QQueryFilter filter                  = queryInput.getFilter();
+         if(filter != null && tableDetails.getCardinality().equals(Cardinality.ONE))
+         {
+            if(filter.getCriteria() != null && filter.getCriteria().size() == 1)
+            {
+               QFilterCriteria criteria = filter.getCriteria().get(0);
+               if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS))
+               {
+                  requestedPath = ValueUtils.getValueAsString(criteria.getValues().get(0));
+               }
+            }
+         }
+
+         List<FILE> files = listFiles(table, queryInput.getBackend(), requestedPath);
 
          switch(tableDetails.getCardinality())
          {
@@ -305,6 +321,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
    }
 
 
+
    /***************************************************************************
     **
     ***************************************************************************/
@@ -324,6 +341,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
    }
 
 
+
    /***************************************************************************
     **
     ***************************************************************************/
@@ -382,13 +400,12 @@ public abstract class AbstractBaseFilesystemAction<FILE>
          // if so, remove that criteria here, so that its presence doesn't cause all records to be filtered away //
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          QQueryFilter filterForRecords = queryInput.getFilter();
-         if(filterForRecords != null)
+         // if(filterForRecords != null)
-         {
+         // {
-            filterForRecords = filterForRecords.clone();
+         //    filterForRecords = filterForRecords.clone();
-
+         //    CollectionUtils.nonNullList(filterForRecords.getCriteria())
-            CollectionUtils.nonNullList(filterForRecords.getCriteria())
+         //       .removeIf(AbstractBaseFilesystemAction::isPathEqualsCriteria);
-               .removeIf(AbstractBaseFilesystemAction::isPathEqualsCriteria);
+         // }
-         }
 
          if(BackendQueryFilterUtils.doesRecordMatch(filterForRecords, null, record))
          {
@@ -560,6 +577,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
    }
 
 
+
    /***************************************************************************
     ** Method that subclasses can override to add post-action things (e.g., closing resources)
     ***************************************************************************/
@@ -571,6 +589,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
    }
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java

@@ -130,7 +130,11 @@ public class SharedFilesystemBackendModuleUtils
       }
       else
       {
-         throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         // this happens in base class now, like, for query action, so, we think okay to just ignore. //
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         // throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
+         return (true);
       }
    }
 

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java

@@ -41,6 +41,8 @@ import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -111,7 +113,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
     ** List the files for this table.
     *******************************************************************************/
    @Override
-   public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
+   public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException
    {
       try
       {
@@ -130,7 +132,14 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
 
          for(String matchedFile : matchedFiles)
          {
-            if(SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails))
+            boolean isMatch = true;
+            if(StringUtils.hasContent(requestedPath))
+            {
+               QQueryFilter filter = new QQueryFilter(new QFilterCriteria(tableBackendDetails.getFileNameFieldName(), QCriteriaOperator.EQUALS, requestedPath));
+               isMatch = SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails);
+            }
+
+            if(isMatch)
             {
                rs.add(new File(fullPath + File.separatorChar + matchedFile));
             }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java

@@ -33,7 +33,6 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -163,7 +162,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
     ** List the files for a table.
     *******************************************************************************/
    @Override
-   public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
+   public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException
    {
       S3BackendMetaData                     s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase);
       AbstractFilesystemTableBackendDetails tableDetails      = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
@@ -175,7 +174,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
       ////////////////////////////////////////////////////////////////////
       // todo - look at metadata to configure the s3 client here?       //
       ////////////////////////////////////////////////////////////////////
-      return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob, filter, tableDetails);
+      return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob, requestedPath, tableDetails);
    }
 
 

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java

@@ -39,12 +39,8 @@ import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
-import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
-import com.kingsrook.qqq.backend.module.filesystem.base.utils.SharedFilesystemBackendModuleUtils;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
 
@@ -80,7 +76,7 @@ public class S3Utils
     ** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
     ** and also - (possibly) apply a file-name filter (based on the table's details).
     *******************************************************************************/
-   public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
+   public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, String requestedPath, AbstractFilesystemTableBackendDetails tableDetails) throws QException
    {
       //////////////////////////////////////////////////////////////////////////////////////////////////
       // s3 list requests find nothing if the path starts with a /, so strip away any leading slashes //
@@ -96,38 +92,20 @@ public class S3Utils
          prefix = prefix.substring(0, prefix.indexOf('*'));
       }
 
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // for a file-per-record (ONE) table, we may need to apply the filter to listing.                    //
+      // optimization, to avoid listing whole bucket, for use-case where less than a whole bucket is requested //
-      // but for MANY tables, the filtering would be done on the records after they came out of the files. //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(StringUtils.hasContent(requestedPath))
-      boolean useQQueryFilter = false;
-      if(tableDetails != null && Cardinality.ONE.equals(tableDetails.getCardinality()))
       {
-         useQQueryFilter = true;
+         if(!prefix.isEmpty())
-      }
-
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // if there's a filter for single file, make that file name the "prefix" that we send to s3, so we just get back that 1 file. //
-      // as this will be a common case.                                                                                             //
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      if(filter != null && useQQueryFilter)
-      {
-         if(filter.getCriteria() != null && filter.getCriteria().size() == 1)
          {
-            QFilterCriteria criteria = filter.getCriteria().get(0);
+            ///////////////////////////////////////////////////////
-            if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS))
+            // remember, a prefix starting with / finds nothing! //
-            {
+            ///////////////////////////////////////////////////////
-               if(!prefix.isEmpty())
+            prefix += "/";
-               {
-                  ///////////////////////////////////////////////////////
-                  // remember, a prefix starting with / finds nothing! //
-                  ///////////////////////////////////////////////////////
-                  prefix += "/";
-               }
-
-               prefix += criteria.getValues().get(0);
-            }
          }
+
+         prefix += requestedPath;
       }
 
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -203,27 +181,7 @@ public class S3Utils
                continue;
             }
 
-            ///////////////////////////////////////////////////////////////////////////////////
-            // if we're a file-per-record table, and we have a filter, compare the key to it //
-            ///////////////////////////////////////////////////////////////////////////////////
-            if(!SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(key, filter, tableDetails))
-            {
-               continue;
-            }
-
             rs.add(objectSummary);
-
-            /////////////////////////////////////////////////////////////////
-            // if we have a limit, and we've hit it, break out of the loop //
-            /////////////////////////////////////////////////////////////////
-            if(filter != null && useQQueryFilter && filter.getLimit() != null)
-            {
-               if(rs.size() >= filter.getLimit())
-               {
-                  break;
-               }
-            }
-
          }
       }
       while(listObjectsV2Result.isTruncated());

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java

@@ -34,15 +34,13 @@ import java.util.function.Consumer;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
 import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath;
@@ -267,23 +265,14 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
     **
     ***************************************************************************/
    @Override
-   public List<SFTPDirEntryWithPath> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
+   public List<SFTPDirEntryWithPath> listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException
    {
       try
       {
          String fullPath = getFullBasePath(table, backendBase);
-
+         if(StringUtils.hasContent(requestedPath))
-         // todo - move somewhere shared
-         // todo - should all do this?
-         if(filter != null)
          {
-            for(QFilterCriteria criteria : CollectionUtils.nonNullList(filter.getCriteria()))
+            fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + requestedPath + File.separatorChar);
-            {
-               if(isPathEqualsCriteria(criteria))
-               {
-                  fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + criteria.getValues().get(0) + File.separatorChar);
-               }
-            }
          }
 
          List<SFTPDirEntryWithPath> rs = new ArrayList<>();

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java

@@ -26,9 +26,6 @@ import java.io.File;
 import java.io.IOException;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -41,7 +38,6 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
@@ -97,55 +93,59 @@ public class FilesystemBackendModuleTest
       /////////////////////////////////////////
       // filter for a file name that's found //
       /////////////////////////////////////////
-      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
+      files = abstractFilesystemAction.listFiles(table, backend, "BLOB-2.txt");
       assertEquals(1, files.size());
       assertEquals("BLOB-2.txt", files.get(0).getName());
 
-      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
+      files = abstractFilesystemAction.listFiles(table, backend, "BLOB-1.txt");
       assertEquals(1, files.size());
       assertEquals("BLOB-1.txt", files.get(0).getName());
 
-      ///////////////////////////////////
+      ///////////////////////////
-      // filter for 2 names that exist //
+      // not supported anymore //
-      ///////////////////////////////////
+      ///////////////////////////
-      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
+      // ///////////////////////////////////
-      assertEquals(2, files.size());
+      // // filter for 2 names that exist //
+      // ///////////////////////////////////
+      // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
+      // assertEquals(2, files.size());
 
       /////////////////////////////////////////////
       // filter for a file name that isn't found //
       /////////////////////////////////////////////
-      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
+      files = abstractFilesystemAction.listFiles(table, backend, "NOT-FOUND.txt");
       assertEquals(0, files.size());
 
-      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
+      ///////////////////////////
-      assertEquals(1, files.size());
+      // not supported anymore //
+      ///////////////////////////
+      // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
+      // assertEquals(1, files.size());
 
-      ////////////////////////////////////////////////////
+      ///////////////////////////
-      // 2 criteria, and'ed, and can't match, so find 0 //
+      // not supported anymore //
-      ////////////////////////////////////////////////////
+      ///////////////////////////
-      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
+      // ////////////////////////////////////////////////////
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
+      // // 2 criteria, and'ed, and can't match, so find 0 //
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
+      // ////////////////////////////////////////////////////
-      assertEquals(0, files.size());
+      // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
+      //    new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
+      //    new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
+      // assertEquals(0, files.size());
 
-      //////////////////////////////////////////////////
+      // //////////////////////////////////////////////////
-      // 2 criteria, or'ed, and both match, so find 2 //
+      // // 2 criteria, or'ed, and both match, so find 2 //
-      //////////////////////////////////////////////////
+      // //////////////////////////////////////////////////
-      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
+      // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
+      //    new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
+      //    new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
-         .withBooleanOperator(QQueryFilter.BooleanOperator.OR));
+      //    .withBooleanOperator(QQueryFilter.BooleanOperator.OR));
-      assertEquals(2, files.size());
+      // assertEquals(2, files.size());
 
-      //////////////////////////////////////
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // ensure unsupported filters throw //
+      // note that we used to try unsupported filters here, expecting them to throw - but those are           //
-      //////////////////////////////////////
+      // more-or-less now implemented in the base class's query method, so, no longer expected to throw here. //
-      assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////
-         .rootCause()
-         .hasMessageContaining("Unable to query filesystem table by field");
-      assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
-         .rootCause()
-         .hasMessageContaining("Unable to query filename field using operator");
    }
 
 

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java

@@ -26,9 +26,6 @@ import java.util.List;
 import java.util.UUID;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -38,7 +35,6 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
@@ -77,53 +73,59 @@ public class S3BackendModuleTest extends BaseS3Test
       /////////////////////////////////////////
       // filter for a file name that's found //
       /////////////////////////////////////////
-      files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
+      files = actionBase.listFiles(table, backend, "BLOB-2.txt");
       assertEquals(1, files.size());
       assertThat(files.get(0).getKey()).contains("BLOB-2.txt");
 
-      files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
+      files = actionBase.listFiles(table, backend, "BLOB-1.txt");
       assertEquals(1, files.size());
       assertThat(files.get(0).getKey()).contains("BLOB-1.txt");
 
-      ///////////////////////////////////
+      ///////////////////////////
-      // filter for 2 names that exist //
+      // not supported anymore //
-      ///////////////////////////////////
+      ///////////////////////////
-      files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
+      // ///////////////////////////////////
-      assertEquals(2, files.size());
+      // // filter for 2 names that exist //
+      // ///////////////////////////////////
+      // files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
+      // assertEquals(2, files.size());
 
       /////////////////////////////////////////////
       // filter for a file name that isn't found //
       /////////////////////////////////////////////
-      files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
+      files = actionBase.listFiles(table, backend, "NOT-FOUND.txt");
       assertEquals(0, files.size());
 
-      files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
+      ///////////////////////////
-      assertEquals(1, files.size());
+      // not supported anymore //
+      ///////////////////////////
+      // files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
+      // assertEquals(1, files.size());
 
-      ////////////////////////////////////////////////////
+      ///////////////////////////
-      // 2 criteria, and'ed, and can't match, so find 0 //
+      // not supported anymore //
-      ////////////////////////////////////////////////////
+      ///////////////////////////
-      files = actionBase.listFiles(table, backend, new QQueryFilter(
+      // ////////////////////////////////////////////////////
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
+      // // 2 criteria, and'ed, and can't match, so find 0 //
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
+      // ////////////////////////////////////////////////////
-      assertEquals(0, files.size());
+      // files = actionBase.listFiles(table, backend, new QQueryFilter(
+      //    new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
+      //    new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
+      // assertEquals(0, files.size());
 
-      //////////////////////////////////////////////////
+      // //////////////////////////////////////////////////
-      // 2 criteria, or'ed, and both match, so find 2 //
+      // // 2 criteria, or'ed, and both match, so find 2 //
-      //////////////////////////////////////////////////
+      // //////////////////////////////////////////////////
-      files = actionBase.listFiles(table, backend, new QQueryFilter(
+      // files = actionBase.listFiles(table, backend, new QQueryFilter(
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
+      //    new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
-         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
+      //    new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
-         .withBooleanOperator(QQueryFilter.BooleanOperator.OR));
+      //    .withBooleanOperator(QQueryFilter.BooleanOperator.OR));
-      assertEquals(2, files.size());
+      // assertEquals(2, files.size());
 
-      //////////////////////////////////////
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // ensure unsupported filters throw //
+      // note that we used to try unsupported filters here, expecting them to throw - but those are           //
-      //////////////////////////////////////
+      // more-or-less now implemented in the base class's query method, so, no longer expected to throw here. //
-      assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////
-         .hasMessageContaining("Unable to query filesystem table by field");
-      assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
-         .hasMessageContaining("Unable to query filename field using operator");
    }
 
 

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java

@@ -23,15 +23,11 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
 
 
 import java.util.List;
-import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -62,69 +58,6 @@ class SFTPQueryActionTest extends BaseSFTPTest
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void testQueryWithPath() throws Exception
-   {
-      String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/";
-      try
-      {
-         copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt");
-         copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt");
-
-         QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE)
-            .withFilter(new QQueryFilter(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder")));
-         QueryOutput queryOutput = new QueryAction().execute(queryInput);
-         Assertions.assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows from subfolder path query");
-      }
-      finally
-      {
-         rmrfInContainer(subfolderPath);
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void testQueryWithPathAndNameLike() throws Exception
-   {
-      String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/";
-      try
-      {
-         copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt");
-         copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt");
-         copyFileToContainer("files/testfile.txt", subfolderPath + "/who.txt");
-
-         Map<String, Integer> patternExpectedCountMap = Map.of(
-            "%.txt", 3,
-            "sub%", 2,
-            "%1%", 1,
-            "%", 3,
-            "*", 0
-         );
-
-         for(Map.Entry<String, Integer> entry : patternExpectedCountMap.entrySet())
-         {
-            QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE).withFilter(new QQueryFilter()
-               .withCriteria(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder"))
-               .withCriteria(new QFilterCriteria("baseName", QCriteriaOperator.LIKE, entry.getKey())));
-            QueryOutput queryOutput = new QueryAction().execute(queryInput);
-            Assertions.assertEquals(entry.getValue(), queryOutput.getRecords().size(), "Expected # of rows from subfolder path, baseName like: " + entry.getKey());
-         }
-      }
-      finally
-      {
-         rmrfInContainer(subfolderPath);
-      }
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/