CE-773 Fixing globs for local filesystem by using Files.walkFileTree. Refactored to share filter matching between s3 & local fs.

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java

@@ -0,0 +1,137 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.base.utils;
+
+
+import java.net.URI;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
+
+
+/*******************************************************************************
+ ** utility methods shared by s3 & local-filesystem utils classes
+ *******************************************************************************/
+public class SharedFilesystemBackendModuleUtils
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static boolean doesFilePathMatchFilter(String filePath, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
+   {
+      if(filter == null || !filter.hasAnyCriteria())
+      {
+         return (true);
+      }
+
+      if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
+      {
+         ///////////////////////////////
+         // todo - well, we could ... //
+         ///////////////////////////////
+         throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time."));
+      }
+
+      Path path = Path.of(URI.create("file:///" + filePath));
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      // foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      for(QFilterCriteria criteria : filter.getCriteria())
+      {
+         boolean matches = doesFilePathMatchOneCriteria(criteria, tableDetails, path);
+
+         if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
+         {
+            ////////////////////////////////////////////////////////////////////////////////
+            // if it's not a match, and it's an AND filter, then the whole thing is false //
+            ////////////////////////////////////////////////////////////////////////////////
+            return (false);
+         }
+
+         if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
+         {
+            ////////////////////////////////////////////////////////////
+            // if it's an OR filter, and we've a match, return a true //
+            ////////////////////////////////////////////////////////////
+            return (true);
+         }
+      }
+
+      //////////////////////////////////////////////////////////////////////
+      // if we didn't return above, return now                            //
+      // for an OR - if we didn't find something true, then return false. //
+      // else, an AND - if we didn't find a false, we can return true.    //
+      //////////////////////////////////////////////////////////////////////
+      if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
+      {
+         return (false);
+      }
+
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static boolean doesFilePathMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException
+   {
+      if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
+      {
+         if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
+         {
+            return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path));
+         }
+         else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
+         {
+            boolean anyMatch = false;
+            for(int i = 0; i < criteria.getValues().size(); i++)
+            {
+               if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path))
+               {
+                  anyMatch = true;
+                  break;
+               }
+            }
+
+            return (anyMatch);
+         }
+         else
+         {
+            throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
+         }
+      }
+      else
+      {
+         throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
+      }
+   }
+
+}

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java

@@ -23,36 +23,32 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
 
 
 import java.io.File;
-import java.io.FileFilter;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
-import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
-import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
+import com.kingsrook.qqq.backend.module.filesystem.base.utils.SharedFilesystemBackendModuleUtils;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOCase;
-import org.apache.commons.io.filefilter.AndFileFilter;
-import org.apache.commons.io.filefilter.NameFileFilter;
-import org.apache.commons.io.filefilter.OrFileFilter;
-import org.apache.commons.io.filefilter.TrueFileFilter;
-import org.apache.commons.io.filefilter.WildcardFileFilter;
 
 
 /*******************************************************************************
@@ -69,80 +65,70 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
     *******************************************************************************/
    @Override
    public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
+   {
+      try
       {
          String fullPath  = getFullBasePath(table, backendBase);
          File   directory = new File(fullPath);
-      File[] files     = null;
 
          AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
 
-      FileFilter fileFilter = TrueFileFilter.INSTANCE;
+         String pattern = "regex:.*";
-
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // if each file is its own record (ONE), then we may need to do filtering of the directory listing based on the input filter //
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      if(Cardinality.ONE.equals(tableBackendDetails.getCardinality()))
-      {
-         if(filter != null && filter.hasAnyCriteria())
-         {
-            if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
-            {
-               ///////////////////////////////////////////////////////////////////////
-               // todo - well, we could - just build up a tree of and's and or's... //
-               ///////////////////////////////////////////////////////////////////////
-               throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time."));
-            }
-
-            List<FileFilter> fileFilterList = new ArrayList<>();
-            for(QFilterCriteria criteria : filter.getCriteria())
-            {
-               if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
-               {
-                  if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
-                  {
-                     fileFilterList.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(0))));
-                  }
-                  else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
-                  {
-                     List<NameFileFilter> nameInFilters = new ArrayList<>();
-                     for(int i = 0; i < criteria.getValues().size(); i++)
-                     {
-                        nameInFilters.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(i))));
-                     }
-                     fileFilterList.add(new OrFileFilter(nameInFilters));
-                  }
-                  else
-                  {
-                     throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
-                  }
-               }
-               else
-               {
-                  throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
-               }
-            }
-
-            fileFilter = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? new AndFileFilter(fileFilterList) : new OrFileFilter(fileFilterList);
-         }
-      }
-
-      ///////////////////////////////////////////////////////////////////////////////////////////
-      // if the table has a glob specified, add it as an AND to the filter built to this point //
-      ///////////////////////////////////////////////////////////////////////////////////////////
          if(StringUtils.hasContent(tableBackendDetails.getGlob()))
          {
-         WildcardFileFilter globFilenameFilter = new WildcardFileFilter(tableBackendDetails.getGlob(), IOCase.INSENSITIVE);
+            pattern = "glob:" + tableBackendDetails.getGlob();
-         fileFilter = new AndFileFilter(List.of(globFilenameFilter, fileFilter));
          }
+         List<String> matchedFiles = recursivelyListFilesMatchingPattern(directory.toPath(), pattern, backendBase, table);
+         List<File>   rs           = new ArrayList<>();
 
-      files = directory.listFiles(fileFilter);
+         for(String matchedFile : matchedFiles)
-
-      if(files == null)
          {
-         return Collections.emptyList();
+            if(SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails))
+            {
+               rs.add(new File(fullPath + File.separatorChar + matchedFile));
+            }
          }
 
-      return (Arrays.stream(files).filter(File::isFile).toList());
+         return (rs);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error searching files", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Credit: https://www.baeldung.com/java-files-match-wildcard-strings
+    *******************************************************************************/
+   List<String> recursivelyListFilesMatchingPattern(Path rootDir, String pattern, QBackendMetaData backend, QTableMetaData table) throws IOException
+   {
+      List<String> matchesList = new ArrayList<>();
+
+      FileVisitor<Path> matcherVisitor = new SimpleFileVisitor<>()
+      {
+         @Override
+         public FileVisitResult visitFile(Path file, BasicFileAttributes attribs)
+         {
+            FileSystem  fs      = FileSystems.getDefault();
+            PathMatcher matcher = fs.getPathMatcher(pattern);
+            Path        path    = Path.of(stripBackendAndTableBasePathsFromFileName(file.toAbsolutePath().toString(), backend, table));
+
+            if(matcher.matches(path))
+            {
+               matchesList.add(path.toString());
+            }
+            return FileVisitResult.CONTINUE;
+         }
+      };
+
+      if(rootDir.toFile().exists())
+      {
+         Files.walkFileTree(rootDir, matcherVisitor);
+      }
+
+      return matchesList;
    }
 
 

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java

@@ -42,9 +42,9 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
+import com.kingsrook.qqq.backend.module.filesystem.base.utils.SharedFilesystemBackendModuleUtils;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
 
@@ -198,7 +198,7 @@ public class S3Utils
             ///////////////////////////////////////////////////////////////////////////////////
             // if we're a file-per-record table, and we have a filter, compare the key to it //
             ///////////////////////////////////////////////////////////////////////////////////
-            if(!doesObjectKeyMatchFilter(key, filter, tableDetails))
+            if(!SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(key, filter, tableDetails))
             {
                continue;
             }
@@ -225,103 +225,6 @@ public class S3Utils
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private boolean doesObjectKeyMatchFilter(String key, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
-   {
-      if(filter == null || !filter.hasAnyCriteria())
-      {
-         return (true);
-      }
-
-      if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
-      {
-         ///////////////////////////////
-         // todo - well, we could ... //
-         ///////////////////////////////
-         throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time."));
-      }
-
-      Path path = Path.of(URI.create("file:///" + key));
-
-      ////////////////////////////////////////////////////////////////////////////////////////////////////
-      // foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches //
-      ////////////////////////////////////////////////////////////////////////////////////////////////////
-      for(QFilterCriteria criteria : filter.getCriteria())
-      {
-         boolean matches = doesObjectKeyMatchOneCriteria(criteria, tableDetails, path);
-
-         if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
-         {
-            ////////////////////////////////////////////////////////////////////////////////
-            // if it's not a match, and it's an AND filter, then the whole thing is false //
-            ////////////////////////////////////////////////////////////////////////////////
-            return (false);
-         }
-
-         if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
-         {
-            ////////////////////////////////////////////////////////////
-            // if it's an OR filter, and we've a match, return a true //
-            ////////////////////////////////////////////////////////////
-            return (true);
-         }
-      }
-
-      //////////////////////////////////////////////////////////////////////
-      // if we didn't return above, return now                            //
-      // for an OR - if we didn't find something true, then return false. //
-      // else, an AND - if we didn't find a false, we can return true.    //
-      //////////////////////////////////////////////////////////////////////
-      if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
-      {
-         return (false);
-      }
-
-      return (true);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static boolean doesObjectKeyMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException
-   {
-      if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
-      {
-         if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
-         {
-            return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path));
-         }
-         else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
-         {
-            boolean anyMatch = false;
-            for(int i = 0; i < criteria.getValues().size(); i++)
-            {
-               if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path))
-               {
-                  anyMatch = true;
-                  break;
-               }
-            }
-
-            return (anyMatch);
-         }
-         else
-         {
-            throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
-         }
-      }
-      else
-      {
-         throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
-      }
-   }
-
-
-
    /*******************************************************************************
     ** Get the contents (as an InputStream) for an object in s3
     *******************************************************************************/

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java

@@ -141,8 +141,10 @@ public class FilesystemBackendModuleTest
       // ensure unsupported filters throw //
       //////////////////////////////////////
       assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
+         .rootCause()
          .hasMessageContaining("Unable to query filesystem table by field");
       assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
+         .rootCause()
          .hasMessageContaining("Unable to query filename field using operator");
    }