QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-773 Fixing globs for local filesystem by using Files.walkFileTree. Refactored to share filter matching between s3 & local fs.

Browse Source
This commit is contained in:
darin.kelkhoff
2023-12-29 08:20:38 -06:00
parent 6e1ea5c8f1
commit 688e221f9a
4 changed files with 203 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java Normal file
View File

@ -0,0 +1,137 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base.utils;
import java.net.URI;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
/*******************************************************************************
** utility methods shared by s3 & local-filesystem utils classes
*******************************************************************************/
public class SharedFilesystemBackendModuleUtils
{
/*******************************************************************************
**
*******************************************************************************/
public static boolean doesFilePathMatchFilter(String filePath, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
{
if(filter == null || !filter.hasAnyCriteria())
{
return (true);
}
if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
{
///////////////////////////////
// todo - well, we could ... //
///////////////////////////////
throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time."));
}
Path path = Path.of(URI.create("file:///" + filePath));
////////////////////////////////////////////////////////////////////////////////////////////////////
// foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches //
////////////////////////////////////////////////////////////////////////////////////////////////////
for(QFilterCriteria criteria : filter.getCriteria())
{
boolean matches = doesFilePathMatchOneCriteria(criteria, tableDetails, path);
if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
{
////////////////////////////////////////////////////////////////////////////////
// if it's not a match, and it's an AND filter, then the whole thing is false //
////////////////////////////////////////////////////////////////////////////////
return (false);
}
if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
{
////////////////////////////////////////////////////////////
// if it's an OR filter, and we've a match, return a true //
////////////////////////////////////////////////////////////
return (true);
}
}
//////////////////////////////////////////////////////////////////////
// if we didn't return above, return now //
// for an OR - if we didn't find something true, then return false. //
// else, an AND - if we didn't find a false, we can return true. //
//////////////////////////////////////////////////////////////////////
if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
{
return (false);
}
return (true);
}
/*******************************************************************************
**
*******************************************************************************/
private static boolean doesFilePathMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException
{
if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
{
if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
{
return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path));
}
else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
{
boolean anyMatch = false;
for(int i = 0; i < criteria.getValues().size(); i++)
{
if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path))
{
anyMatch = true;
break;
}
}
return (anyMatch);
}
else
{
throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
}
}
else
{
throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
}
}
}

138
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java
View File

@ -23,36 +23,32 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.utils.SharedFilesystemBackendModuleUtils;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOCase;
import org.apache.commons.io.filefilter.AndFileFilter;
import org.apache.commons.io.filefilter.NameFileFilter;
import org.apache.commons.io.filefilter.OrFileFilter;
import org.apache.commons.io.filefilter.TrueFileFilter;
import org.apache.commons.io.filefilter.WildcardFileFilter;
/*******************************************************************************
@ -70,79 +66,69 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
@Override
public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
{
String fullPath = getFullBasePath(table, backendBase);
File directory = new File(fullPath);
File[] files = null;
AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
FileFilter fileFilter = TrueFileFilter.INSTANCE;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if each file is its own record (ONE), then we may need to do filtering of the directory listing based on the input filter //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(Cardinality.ONE.equals(tableBackendDetails.getCardinality()))
try
{
if(filter != null && filter.hasAnyCriteria())
String fullPath = getFullBasePath(table, backendBase);
File directory = new File(fullPath);
AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
String pattern = "regex:.*";
if(StringUtils.hasContent(tableBackendDetails.getGlob()))
{
if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
{
///////////////////////////////////////////////////////////////////////
// todo - well, we could - just build up a tree of and's and or's... //
///////////////////////////////////////////////////////////////////////
throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time."));
}
List<FileFilter> fileFilterList = new ArrayList<>();
for(QFilterCriteria criteria : filter.getCriteria())
{
if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
{
if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
{
fileFilterList.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(0))));
}
else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
{
List<NameFileFilter> nameInFilters = new ArrayList<>();
for(int i = 0; i < criteria.getValues().size(); i++)
{
nameInFilters.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(i))));
}
fileFilterList.add(new OrFileFilter(nameInFilters));
}
else
{
throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
}
}
else
{
throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
}
}
fileFilter = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? new AndFileFilter(fileFilterList) : new OrFileFilter(fileFilterList);
pattern = "glob:" + tableBackendDetails.getGlob();
}
}
List<String> matchedFiles = recursivelyListFilesMatchingPattern(directory.toPath(), pattern, backendBase, table);
List<File> rs = new ArrayList<>();
///////////////////////////////////////////////////////////////////////////////////////////
// if the table has a glob specified, add it as an AND to the filter built to this point //
///////////////////////////////////////////////////////////////////////////////////////////
if(StringUtils.hasContent(tableBackendDetails.getGlob()))
for(String matchedFile : matchedFiles)
{
if(SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails))
{
rs.add(new File(fullPath + File.separatorChar + matchedFile));
}
}
return (rs);
}
catch(Exception e)
{
WildcardFileFilter globFilenameFilter = new WildcardFileFilter(tableBackendDetails.getGlob(), IOCase.INSENSITIVE);
fileFilter = new AndFileFilter(List.of(globFilenameFilter, fileFilter));
throw (new QException("Error searching files", e));
}
}
files = directory.listFiles(fileFilter);
if(files == null)
/*******************************************************************************
** Credit: https://www.baeldung.com/java-files-match-wildcard-strings
*******************************************************************************/
List<String> recursivelyListFilesMatchingPattern(Path rootDir, String pattern, QBackendMetaData backend, QTableMetaData table) throws IOException
{
List<String> matchesList = new ArrayList<>();
FileVisitor<Path> matcherVisitor = new SimpleFileVisitor<>()
{
return Collections.emptyList();
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attribs)
{
FileSystem fs = FileSystems.getDefault();
PathMatcher matcher = fs.getPathMatcher(pattern);
Path path = Path.of(stripBackendAndTableBasePathsFromFileName(file.toAbsolutePath().toString(), backend, table));
if(matcher.matches(path))
{
matchesList.add(path.toString());
}
return FileVisitResult.CONTINUE;
}
};
if(rootDir.toFile().exists())
{
Files.walkFileTree(rootDir, matcherVisitor);
}
return (Arrays.stream(files).filter(File::isFile).toList());
return matchesList;
}

101
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java
View File

@ -42,9 +42,9 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.utils.SharedFilesystemBackendModuleUtils;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
@ -198,7 +198,7 @@ public class S3Utils
///////////////////////////////////////////////////////////////////////////////////
// if we're a file-per-record table, and we have a filter, compare the key to it //
///////////////////////////////////////////////////////////////////////////////////
if(!doesObjectKeyMatchFilter(key, filter, tableDetails))
if(!SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(key, filter, tableDetails))
{
continue;
}
@ -225,103 +225,6 @@ public class S3Utils
/*******************************************************************************
**
*******************************************************************************/
private boolean doesObjectKeyMatchFilter(String key, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
{
if(filter == null || !filter.hasAnyCriteria())
{
return (true);
}
if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
{
///////////////////////////////
// todo - well, we could ... //
///////////////////////////////
throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time."));
}
Path path = Path.of(URI.create("file:///" + key));
////////////////////////////////////////////////////////////////////////////////////////////////////
// foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches //
////////////////////////////////////////////////////////////////////////////////////////////////////
for(QFilterCriteria criteria : filter.getCriteria())
{
boolean matches = doesObjectKeyMatchOneCriteria(criteria, tableDetails, path);
if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
{
////////////////////////////////////////////////////////////////////////////////
// if it's not a match, and it's an AND filter, then the whole thing is false //
////////////////////////////////////////////////////////////////////////////////
return (false);
}
if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
{
////////////////////////////////////////////////////////////
// if it's an OR filter, and we've a match, return a true //
////////////////////////////////////////////////////////////
return (true);
}
}
//////////////////////////////////////////////////////////////////////
// if we didn't return above, return now //
// for an OR - if we didn't find something true, then return false. //
// else, an AND - if we didn't find a false, we can return true. //
//////////////////////////////////////////////////////////////////////
if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
{
return (false);
}
return (true);
}
/*******************************************************************************
**
*******************************************************************************/
private static boolean doesObjectKeyMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException
{
if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
{
if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
{
return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path));
}
else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
{
boolean anyMatch = false;
for(int i = 0; i < criteria.getValues().size(); i++)
{
if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path))
{
anyMatch = true;
break;
}
}
return (anyMatch);
}
else
{
throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
}
}
else
{
throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
}
}
/*******************************************************************************
** Get the contents (as an InputStream) for an object in s3
*******************************************************************************/

2
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java
View File

@ -141,8 +141,10 @@ public class FilesystemBackendModuleTest
// ensure unsupported filters throw //
//////////////////////////////////////
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
.rootCause()
.hasMessageContaining("Unable to query filesystem table by field");
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
.rootCause()
.hasMessageContaining("Unable to query filename field using operator");
}