diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java new file mode 100644 index 00000000..026386b0 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java @@ -0,0 +1,137 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.utils; + + +import java.net.URI; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; + + +/******************************************************************************* + ** utility methods shared by s3 & local-filesystem utils classes + *******************************************************************************/ +public class SharedFilesystemBackendModuleUtils +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public static boolean doesFilePathMatchFilter(String filePath, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException + { + if(filter == null || !filter.hasAnyCriteria()) + { + return (true); + } + + if(CollectionUtils.nullSafeHasContents(filter.getSubFilters())) + { + /////////////////////////////// + // todo - well, we could ... // + /////////////////////////////// + throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time.")); + } + + Path path = Path.of(URI.create("file:///" + filePath)); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches // + //////////////////////////////////////////////////////////////////////////////////////////////////// + for(QFilterCriteria criteria : filter.getCriteria()) + { + boolean matches = doesFilePathMatchOneCriteria(criteria, tableDetails, path); + + if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator())) + { + //////////////////////////////////////////////////////////////////////////////// + // if it's not a match, and it's an AND filter, then the whole thing is false // + //////////////////////////////////////////////////////////////////////////////// + return (false); + } + + if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator())) + { + //////////////////////////////////////////////////////////// + // if it's an OR filter, and we've a match, return a true // + //////////////////////////////////////////////////////////// + return (true); + } + } + + ////////////////////////////////////////////////////////////////////// + // if we didn't return above, return now // + // for an OR - if we didn't find something true, then return false. // + // else, an AND - if we didn't find a false, we can return true. // + ////////////////////////////////////////////////////////////////////// + if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator())) + { + return (false); + } + + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static boolean doesFilePathMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException + { + if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName())) + { + if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1) + { + return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path)); + } + else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty()) + { + boolean anyMatch = false; + for(int i = 0; i < criteria.getValues().size(); i++) + { + if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path)) + { + anyMatch = true; + break; + } + } + + return (anyMatch); + } + else + { + throw (new QException("Unable to query filename field using operator: " + criteria.getOperator())); + } + } + else + { + throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName())); + } + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index 3e9f1919..df687e90 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -23,36 +23,32 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions; import java.io.File; -import java.io.FileFilter; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; -import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; +import com.kingsrook.qqq.backend.module.filesystem.base.utils.SharedFilesystemBackendModuleUtils; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOCase; -import org.apache.commons.io.filefilter.AndFileFilter; -import org.apache.commons.io.filefilter.NameFileFilter; -import org.apache.commons.io.filefilter.OrFileFilter; -import org.apache.commons.io.filefilter.TrueFileFilter; -import org.apache.commons.io.filefilter.WildcardFileFilter; /******************************************************************************* @@ -70,79 +66,69 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction @Override public List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException { - String fullPath = getFullBasePath(table, backendBase); - File directory = new File(fullPath); - File[] files = null; - - AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); - - FileFilter fileFilter = TrueFileFilter.INSTANCE; - - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // if each file is its own record (ONE), then we may need to do filtering of the directory listing based on the input filter // - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(Cardinality.ONE.equals(tableBackendDetails.getCardinality())) + try { - if(filter != null && filter.hasAnyCriteria()) + String fullPath = getFullBasePath(table, backendBase); + File directory = new File(fullPath); + + AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); + + String pattern = "regex:.*"; + if(StringUtils.hasContent(tableBackendDetails.getGlob())) { - if(CollectionUtils.nullSafeHasContents(filter.getSubFilters())) - { - /////////////////////////////////////////////////////////////////////// - // todo - well, we could - just build up a tree of and's and or's... // - /////////////////////////////////////////////////////////////////////// - throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time.")); - } - - List fileFilterList = new ArrayList<>(); - for(QFilterCriteria criteria : filter.getCriteria()) - { - if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName())) - { - if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1) - { - fileFilterList.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(0)))); - } - else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty()) - { - List nameInFilters = new ArrayList<>(); - for(int i = 0; i < criteria.getValues().size(); i++) - { - nameInFilters.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(i)))); - } - fileFilterList.add(new OrFileFilter(nameInFilters)); - } - else - { - throw (new QException("Unable to query filename field using operator: " + criteria.getOperator())); - } - } - else - { - throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName())); - } - } - - fileFilter = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? new AndFileFilter(fileFilterList) : new OrFileFilter(fileFilterList); + pattern = "glob:" + tableBackendDetails.getGlob(); } - } + List matchedFiles = recursivelyListFilesMatchingPattern(directory.toPath(), pattern, backendBase, table); + List rs = new ArrayList<>(); - /////////////////////////////////////////////////////////////////////////////////////////// - // if the table has a glob specified, add it as an AND to the filter built to this point // - /////////////////////////////////////////////////////////////////////////////////////////// - if(StringUtils.hasContent(tableBackendDetails.getGlob())) + for(String matchedFile : matchedFiles) + { + if(SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails)) + { + rs.add(new File(fullPath + File.separatorChar + matchedFile)); + } + } + + return (rs); + } + catch(Exception e) { - WildcardFileFilter globFilenameFilter = new WildcardFileFilter(tableBackendDetails.getGlob(), IOCase.INSENSITIVE); - fileFilter = new AndFileFilter(List.of(globFilenameFilter, fileFilter)); + throw (new QException("Error searching files", e)); } + } - files = directory.listFiles(fileFilter); - if(files == null) + + /******************************************************************************* + ** Credit: https://www.baeldung.com/java-files-match-wildcard-strings + *******************************************************************************/ + List recursivelyListFilesMatchingPattern(Path rootDir, String pattern, QBackendMetaData backend, QTableMetaData table) throws IOException + { + List matchesList = new ArrayList<>(); + + FileVisitor matcherVisitor = new SimpleFileVisitor<>() { - return Collections.emptyList(); + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attribs) + { + FileSystem fs = FileSystems.getDefault(); + PathMatcher matcher = fs.getPathMatcher(pattern); + Path path = Path.of(stripBackendAndTableBasePathsFromFileName(file.toAbsolutePath().toString(), backend, table)); + + if(matcher.matches(path)) + { + matchesList.add(path.toString()); + } + return FileVisitResult.CONTINUE; + } + }; + + if(rootDir.toFile().exists()) + { + Files.walkFileTree(rootDir, matcherVisitor); } - return (Arrays.stream(files).filter(File::isFile).toList()); + return matchesList; } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java index c54e1d20..2a22dbe9 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java @@ -42,9 +42,9 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; +import com.kingsrook.qqq.backend.module.filesystem.base.utils.SharedFilesystemBackendModuleUtils; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction; @@ -198,7 +198,7 @@ public class S3Utils /////////////////////////////////////////////////////////////////////////////////// // if we're a file-per-record table, and we have a filter, compare the key to it // /////////////////////////////////////////////////////////////////////////////////// - if(!doesObjectKeyMatchFilter(key, filter, tableDetails)) + if(!SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(key, filter, tableDetails)) { continue; } @@ -225,103 +225,6 @@ public class S3Utils - /******************************************************************************* - ** - *******************************************************************************/ - private boolean doesObjectKeyMatchFilter(String key, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException - { - if(filter == null || !filter.hasAnyCriteria()) - { - return (true); - } - - if(CollectionUtils.nullSafeHasContents(filter.getSubFilters())) - { - /////////////////////////////// - // todo - well, we could ... // - /////////////////////////////// - throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time.")); - } - - Path path = Path.of(URI.create("file:///" + key)); - - //////////////////////////////////////////////////////////////////////////////////////////////////// - // foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches // - //////////////////////////////////////////////////////////////////////////////////////////////////// - for(QFilterCriteria criteria : filter.getCriteria()) - { - boolean matches = doesObjectKeyMatchOneCriteria(criteria, tableDetails, path); - - if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator())) - { - //////////////////////////////////////////////////////////////////////////////// - // if it's not a match, and it's an AND filter, then the whole thing is false // - //////////////////////////////////////////////////////////////////////////////// - return (false); - } - - if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator())) - { - //////////////////////////////////////////////////////////// - // if it's an OR filter, and we've a match, return a true // - //////////////////////////////////////////////////////////// - return (true); - } - } - - ////////////////////////////////////////////////////////////////////// - // if we didn't return above, return now // - // for an OR - if we didn't find something true, then return false. // - // else, an AND - if we didn't find a false, we can return true. // - ////////////////////////////////////////////////////////////////////// - if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator())) - { - return (false); - } - - return (true); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private static boolean doesObjectKeyMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException - { - if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName())) - { - if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1) - { - return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path)); - } - else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty()) - { - boolean anyMatch = false; - for(int i = 0; i < criteria.getValues().size(); i++) - { - if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path)) - { - anyMatch = true; - break; - } - } - - return (anyMatch); - } - else - { - throw (new QException("Unable to query filename field using operator: " + criteria.getOperator())); - } - } - else - { - throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName())); - } - } - - - /******************************************************************************* ** Get the contents (as an InputStream) for an object in s3 *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java index f659b265..6faf15bc 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java @@ -141,8 +141,10 @@ public class FilesystemBackendModuleTest // ensure unsupported filters throw // ////////////////////////////////////// assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42)))) + .rootCause() .hasMessageContaining("Unable to query filesystem table by field"); assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK)))) + .rootCause() .hasMessageContaining("Unable to query filename field using operator"); }