mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 05:01:07 +00:00
CE-773 Fixing globs for local filesystem by using Files.walkFileTree. Refactored to share filter matching between s3 & local fs.
This commit is contained in:
@ -0,0 +1,137 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2023. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.base.utils;
|
||||
|
||||
|
||||
import java.net.URI;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.nio.file.Path;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** utility methods shared by s3 & local-filesystem utils classes
|
||||
*******************************************************************************/
|
||||
public class SharedFilesystemBackendModuleUtils
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static boolean doesFilePathMatchFilter(String filePath, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
|
||||
{
|
||||
if(filter == null || !filter.hasAnyCriteria())
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
|
||||
if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
|
||||
{
|
||||
///////////////////////////////
|
||||
// todo - well, we could ... //
|
||||
///////////////////////////////
|
||||
throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time."));
|
||||
}
|
||||
|
||||
Path path = Path.of(URI.create("file:///" + filePath));
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
for(QFilterCriteria criteria : filter.getCriteria())
|
||||
{
|
||||
boolean matches = doesFilePathMatchOneCriteria(criteria, tableDetails, path);
|
||||
|
||||
if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// if it's not a match, and it's an AND filter, then the whole thing is false //
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
return (false);
|
||||
}
|
||||
|
||||
if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
|
||||
{
|
||||
////////////////////////////////////////////////////////////
|
||||
// if it's an OR filter, and we've a match, return a true //
|
||||
////////////////////////////////////////////////////////////
|
||||
return (true);
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// if we didn't return above, return now //
|
||||
// for an OR - if we didn't find something true, then return false. //
|
||||
// else, an AND - if we didn't find a false, we can return true. //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static boolean doesFilePathMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException
|
||||
{
|
||||
if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
|
||||
{
|
||||
if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
|
||||
{
|
||||
return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path));
|
||||
}
|
||||
else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
|
||||
{
|
||||
boolean anyMatch = false;
|
||||
for(int i = 0; i < criteria.getValues().size(); i++)
|
||||
{
|
||||
if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path))
|
||||
{
|
||||
anyMatch = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return (anyMatch);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -23,36 +23,32 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
||||
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileFilter;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.FileSystem;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.nio.file.FileVisitResult;
|
||||
import java.nio.file.FileVisitor;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.PathMatcher;
|
||||
import java.nio.file.SimpleFileVisitor;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.utils.SharedFilesystemBackendModuleUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.IOCase;
|
||||
import org.apache.commons.io.filefilter.AndFileFilter;
|
||||
import org.apache.commons.io.filefilter.NameFileFilter;
|
||||
import org.apache.commons.io.filefilter.OrFileFilter;
|
||||
import org.apache.commons.io.filefilter.TrueFileFilter;
|
||||
import org.apache.commons.io.filefilter.WildcardFileFilter;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -69,80 +65,70 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
String fullPath = getFullBasePath(table, backendBase);
|
||||
File directory = new File(fullPath);
|
||||
File[] files = null;
|
||||
|
||||
AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
|
||||
|
||||
FileFilter fileFilter = TrueFileFilter.INSTANCE;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// if each file is its own record (ONE), then we may need to do filtering of the directory listing based on the input filter //
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(Cardinality.ONE.equals(tableBackendDetails.getCardinality()))
|
||||
{
|
||||
if(filter != null && filter.hasAnyCriteria())
|
||||
{
|
||||
if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// todo - well, we could - just build up a tree of and's and or's... //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time."));
|
||||
}
|
||||
|
||||
List<FileFilter> fileFilterList = new ArrayList<>();
|
||||
for(QFilterCriteria criteria : filter.getCriteria())
|
||||
{
|
||||
if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
|
||||
{
|
||||
if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
|
||||
{
|
||||
fileFilterList.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(0))));
|
||||
}
|
||||
else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
|
||||
{
|
||||
List<NameFileFilter> nameInFilters = new ArrayList<>();
|
||||
for(int i = 0; i < criteria.getValues().size(); i++)
|
||||
{
|
||||
nameInFilters.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(i))));
|
||||
}
|
||||
fileFilterList.add(new OrFileFilter(nameInFilters));
|
||||
}
|
||||
else
|
||||
{
|
||||
throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
|
||||
}
|
||||
}
|
||||
|
||||
fileFilter = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? new AndFileFilter(fileFilterList) : new OrFileFilter(fileFilterList);
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////
|
||||
// if the table has a glob specified, add it as an AND to the filter built to this point //
|
||||
///////////////////////////////////////////////////////////////////////////////////////////
|
||||
String pattern = "regex:.*";
|
||||
if(StringUtils.hasContent(tableBackendDetails.getGlob()))
|
||||
{
|
||||
WildcardFileFilter globFilenameFilter = new WildcardFileFilter(tableBackendDetails.getGlob(), IOCase.INSENSITIVE);
|
||||
fileFilter = new AndFileFilter(List.of(globFilenameFilter, fileFilter));
|
||||
pattern = "glob:" + tableBackendDetails.getGlob();
|
||||
}
|
||||
List<String> matchedFiles = recursivelyListFilesMatchingPattern(directory.toPath(), pattern, backendBase, table);
|
||||
List<File> rs = new ArrayList<>();
|
||||
|
||||
files = directory.listFiles(fileFilter);
|
||||
|
||||
if(files == null)
|
||||
for(String matchedFile : matchedFiles)
|
||||
{
|
||||
return Collections.emptyList();
|
||||
if(SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails))
|
||||
{
|
||||
rs.add(new File(fullPath + File.separatorChar + matchedFile));
|
||||
}
|
||||
}
|
||||
|
||||
return (Arrays.stream(files).filter(File::isFile).toList());
|
||||
return (rs);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (new QException("Error searching files", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Credit: https://www.baeldung.com/java-files-match-wildcard-strings
|
||||
*******************************************************************************/
|
||||
List<String> recursivelyListFilesMatchingPattern(Path rootDir, String pattern, QBackendMetaData backend, QTableMetaData table) throws IOException
|
||||
{
|
||||
List<String> matchesList = new ArrayList<>();
|
||||
|
||||
FileVisitor<Path> matcherVisitor = new SimpleFileVisitor<>()
|
||||
{
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attribs)
|
||||
{
|
||||
FileSystem fs = FileSystems.getDefault();
|
||||
PathMatcher matcher = fs.getPathMatcher(pattern);
|
||||
Path path = Path.of(stripBackendAndTableBasePathsFromFileName(file.toAbsolutePath().toString(), backend, table));
|
||||
|
||||
if(matcher.matches(path))
|
||||
{
|
||||
matchesList.add(path.toString());
|
||||
}
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
};
|
||||
|
||||
if(rootDir.toFile().exists())
|
||||
{
|
||||
Files.walkFileTree(rootDir, matcherVisitor);
|
||||
}
|
||||
|
||||
return matchesList;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -42,9 +42,9 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.utils.SharedFilesystemBackendModuleUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
|
||||
|
||||
@ -198,7 +198,7 @@ public class S3Utils
|
||||
///////////////////////////////////////////////////////////////////////////////////
|
||||
// if we're a file-per-record table, and we have a filter, compare the key to it //
|
||||
///////////////////////////////////////////////////////////////////////////////////
|
||||
if(!doesObjectKeyMatchFilter(key, filter, tableDetails))
|
||||
if(!SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(key, filter, tableDetails))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
@ -225,103 +225,6 @@ public class S3Utils
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private boolean doesObjectKeyMatchFilter(String key, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
|
||||
{
|
||||
if(filter == null || !filter.hasAnyCriteria())
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
|
||||
if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
|
||||
{
|
||||
///////////////////////////////
|
||||
// todo - well, we could ... //
|
||||
///////////////////////////////
|
||||
throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time."));
|
||||
}
|
||||
|
||||
Path path = Path.of(URI.create("file:///" + key));
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
for(QFilterCriteria criteria : filter.getCriteria())
|
||||
{
|
||||
boolean matches = doesObjectKeyMatchOneCriteria(criteria, tableDetails, path);
|
||||
|
||||
if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// if it's not a match, and it's an AND filter, then the whole thing is false //
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
return (false);
|
||||
}
|
||||
|
||||
if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
|
||||
{
|
||||
////////////////////////////////////////////////////////////
|
||||
// if it's an OR filter, and we've a match, return a true //
|
||||
////////////////////////////////////////////////////////////
|
||||
return (true);
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// if we didn't return above, return now //
|
||||
// for an OR - if we didn't find something true, then return false. //
|
||||
// else, an AND - if we didn't find a false, we can return true. //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static boolean doesObjectKeyMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException
|
||||
{
|
||||
if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
|
||||
{
|
||||
if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
|
||||
{
|
||||
return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path));
|
||||
}
|
||||
else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
|
||||
{
|
||||
boolean anyMatch = false;
|
||||
for(int i = 0; i < criteria.getValues().size(); i++)
|
||||
{
|
||||
if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path))
|
||||
{
|
||||
anyMatch = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return (anyMatch);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Get the contents (as an InputStream) for an object in s3
|
||||
*******************************************************************************/
|
||||
|
||||
@ -141,8 +141,10 @@ public class FilesystemBackendModuleTest
|
||||
// ensure unsupported filters throw //
|
||||
//////////////////////////////////////
|
||||
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
|
||||
.rootCause()
|
||||
.hasMessageContaining("Unable to query filesystem table by field");
|
||||
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
|
||||
.rootCause()
|
||||
.hasMessageContaining("Unable to query filename field using operator");
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user