QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-17 20:50:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Simplify file listing by replacing filters with requested paths

Browse Source 
Refactor file listing mechanisms to replace the use of complex query filters with simpler, path-based requests. Updated module-specific implementations and removed unused filtering logic. Updated tests (zombie'ing some)
This commit is contained in:
darin.kelkhoff
2025-02-20 11:41:29 -06:00
parent be4f3c68f0
commit d25eb6ee48
9 changed files with 143 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
View File

@ -57,8 +57,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSett
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil;
import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
@ -111,9 +111,10 @@ public abstract class AbstractBaseFilesystemAction<FILE>
public abstract Instant getFileModifyDate(FILE file);
/*******************************************************************************
** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses.
** List the files for a table - or optionally, just a single file name -
** to be implemented in module-specific subclasses.
*******************************************************************************/
public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException;
public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedSingleFileName) throws QException;
/*******************************************************************************
** Read the contents of a file - to be implemented in module-specific subclasses.
@ -278,11 +279,26 @@ public abstract class AbstractBaseFilesystemAction<FILE>
try
{
QueryOutput queryOutput = new QueryOutput(queryInput);
QTableMetaData table = queryInput.getTable();
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
List<FILE> files = listFiles(table, queryInput.getBackend(), queryInput.getFilter());
QueryOutput queryOutput = new QueryOutput(queryInput);
String requestedPath = null;
QQueryFilter filter = queryInput.getFilter();
if(filter != null && tableDetails.getCardinality().equals(Cardinality.ONE))
{
if(filter.getCriteria() != null && filter.getCriteria().size() == 1)
{
QFilterCriteria criteria = filter.getCriteria().get(0);
if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS))
{
requestedPath = ValueUtils.getValueAsString(criteria.getValues().get(0));
}
}
}
List<FILE> files = listFiles(table, queryInput.getBackend(), requestedPath);
switch(tableDetails.getCardinality())
{
@ -305,6 +321,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
}
/***************************************************************************
**
***************************************************************************/
@ -324,6 +341,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
}
/***************************************************************************
**
***************************************************************************/
@ -382,13 +400,12 @@ public abstract class AbstractBaseFilesystemAction<FILE>
// if so, remove that criteria here, so that its presence doesn't cause all records to be filtered away //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
QQueryFilter filterForRecords = queryInput.getFilter();
if(filterForRecords != null)
{
filterForRecords = filterForRecords.clone();
CollectionUtils.nonNullList(filterForRecords.getCriteria())
.removeIf(AbstractBaseFilesystemAction::isPathEqualsCriteria);
}
// if(filterForRecords != null)
// {
// filterForRecords = filterForRecords.clone();
// CollectionUtils.nonNullList(filterForRecords.getCriteria())
// .removeIf(AbstractBaseFilesystemAction::isPathEqualsCriteria);
// }
if(BackendQueryFilterUtils.doesRecordMatch(filterForRecords, null, record))
{
@ -560,6 +577,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
}
/***************************************************************************
** Method that subclasses can override to add post-action things (e.g., closing resources)
***************************************************************************/
@ -571,6 +589,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
}
/*******************************************************************************
**
*******************************************************************************/

6
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java
View File

@ -130,7 +130,11 @@ public class SharedFilesystemBackendModuleUtils
}
else
{
throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
///////////////////////////////////////////////////////////////////////////////////////////////
// this happens in base class now, like, for query action, so, we think okay to just ignore. //
///////////////////////////////////////////////////////////////////////////////////////////////
// throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
return (true);
}
}

13
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java
View File

@ -41,6 +41,8 @@ import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -111,7 +113,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
** List the files for this table.
*******************************************************************************/
@Override
public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException
{
try
{
@ -130,7 +132,14 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
for(String matchedFile : matchedFiles)
{
if(SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails))
boolean isMatch = true;
if(StringUtils.hasContent(requestedPath))
{
QQueryFilter filter = new QQueryFilter(new QFilterCriteria(tableBackendDetails.getFileNameFieldName(), QCriteriaOperator.EQUALS, requestedPath));
isMatch = SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails);
}
if(isMatch)
{
rs.add(new File(fullPath + File.separatorChar + matchedFile));
}

5
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java
View File

@ -33,7 +33,6 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@ -163,7 +162,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
** List the files for a table.
*******************************************************************************/
@Override
public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException
{
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase);
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
@ -175,7 +174,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
////////////////////////////////////////////////////////////////////
// todo - look at metadata to configure the s3 client here? //
////////////////////////////////////////////////////////////////////
return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob, filter, tableDetails);
return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob, requestedPath, tableDetails);
}

68
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java
View File

@ -39,12 +39,8 @@ import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.utils.SharedFilesystemBackendModuleUtils;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
@ -80,7 +76,7 @@ public class S3Utils
** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
** and also - (possibly) apply a file-name filter (based on the table's details).
*******************************************************************************/
public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, String requestedPath, AbstractFilesystemTableBackendDetails tableDetails) throws QException
{
//////////////////////////////////////////////////////////////////////////////////////////////////
// s3 list requests find nothing if the path starts with a /, so strip away any leading slashes //
@ -96,38 +92,20 @@ public class S3Utils
prefix = prefix.substring(0, prefix.indexOf('*'));
}
///////////////////////////////////////////////////////////////////////////////////////////////////////
// for a file-per-record (ONE) table, we may need to apply the filter to listing. //
// but for MANY tables, the filtering would be done on the records after they came out of the files. //
///////////////////////////////////////////////////////////////////////////////////////////////////////
boolean useQQueryFilter = false;
if(tableDetails != null && Cardinality.ONE.equals(tableDetails.getCardinality()))
///////////////////////////////////////////////////////////////////////////////////////////////////////////
// optimization, to avoid listing whole bucket, for use-case where less than a whole bucket is requested //
///////////////////////////////////////////////////////////////////////////////////////////////////////////
if(StringUtils.hasContent(requestedPath))
{
useQQueryFilter = true;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if there's a filter for single file, make that file name the "prefix" that we send to s3, so we just get back that 1 file. //
// as this will be a common case. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(filter != null && useQQueryFilter)
{
if(filter.getCriteria() != null && filter.getCriteria().size() == 1)
if(!prefix.isEmpty())
{
QFilterCriteria criteria = filter.getCriteria().get(0);
if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS))
{
if(!prefix.isEmpty())
{
///////////////////////////////////////////////////////
// remember, a prefix starting with / finds nothing! //
///////////////////////////////////////////////////////
prefix += "/";
}
prefix += criteria.getValues().get(0);
}
///////////////////////////////////////////////////////
// remember, a prefix starting with / finds nothing! //
///////////////////////////////////////////////////////
prefix += "/";
}
prefix += requestedPath;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -203,27 +181,7 @@ public class S3Utils
continue;
}
///////////////////////////////////////////////////////////////////////////////////
// if we're a file-per-record table, and we have a filter, compare the key to it //
///////////////////////////////////////////////////////////////////////////////////
if(!SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(key, filter, tableDetails))
{
continue;
}
rs.add(objectSummary);
/////////////////////////////////////////////////////////////////
// if we have a limit, and we've hit it, break out of the loop //
/////////////////////////////////////////////////////////////////
if(filter != null && useQQueryFilter && filter.getLimit() != null)
{
if(rs.size() >= filter.getLimit())
{
break;
}
}
}
}
while(listObjectsV2Result.isTruncated());

19
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java
View File

@ -34,15 +34,13 @@ import java.util.function.Consumer;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath;
@ -267,23 +265,14 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
**
***************************************************************************/
@Override
public List<SFTPDirEntryWithPath> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
public List<SFTPDirEntryWithPath> listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException
{
try
{
String fullPath = getFullBasePath(table, backendBase);
// todo - move somewhere shared
// todo - should all do this?
if(filter != null)
if(StringUtils.hasContent(requestedPath))
{
for(QFilterCriteria criteria : CollectionUtils.nonNullList(filter.getCriteria()))
{
if(isPathEqualsCriteria(criteria))
{
fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + criteria.getValues().get(0) + File.separatorChar);
}
}
fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + requestedPath + File.separatorChar);
}
List<SFTPDirEntryWithPath> rs = new ArrayList<>();

76
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java
View File

@ -26,9 +26,6 @@ import java.io.File;
import java.io.IOException;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@ -41,7 +38,6 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -97,55 +93,59 @@ public class FilesystemBackendModuleTest
/////////////////////////////////////////
// filter for a file name that's found //
/////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
files = abstractFilesystemAction.listFiles(table, backend, "BLOB-2.txt");
assertEquals(1, files.size());
assertEquals("BLOB-2.txt", files.get(0).getName());
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
files = abstractFilesystemAction.listFiles(table, backend, "BLOB-1.txt");
assertEquals(1, files.size());
assertEquals("BLOB-1.txt", files.get(0).getName());
///////////////////////////////////
// filter for 2 names that exist //
///////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
assertEquals(2, files.size());
///////////////////////////
// not supported anymore //
///////////////////////////
// ///////////////////////////////////
// // filter for 2 names that exist //
// ///////////////////////////////////
// files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
// assertEquals(2, files.size());
/////////////////////////////////////////////
// filter for a file name that isn't found //
/////////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
files = abstractFilesystemAction.listFiles(table, backend, "NOT-FOUND.txt");
assertEquals(0, files.size());
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
assertEquals(1, files.size());
///////////////////////////
// not supported anymore //
///////////////////////////
// files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
// assertEquals(1, files.size());
////////////////////////////////////////////////////
// 2 criteria, and'ed, and can't match, so find 0 //
////////////////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
assertEquals(0, files.size());
///////////////////////////
// not supported anymore //
///////////////////////////
// ////////////////////////////////////////////////////
// // 2 criteria, and'ed, and can't match, so find 0 //
// ////////////////////////////////////////////////////
// files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
// new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
// new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
// assertEquals(0, files.size());
//////////////////////////////////////////////////
// 2 criteria, or'ed, and both match, so find 2 //
//////////////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
.withBooleanOperator(QQueryFilter.BooleanOperator.OR));
assertEquals(2, files.size());
// //////////////////////////////////////////////////
// // 2 criteria, or'ed, and both match, so find 2 //
// //////////////////////////////////////////////////
// files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
// new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
// new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
// .withBooleanOperator(QQueryFilter.BooleanOperator.OR));
// assertEquals(2, files.size());
//////////////////////////////////////
// ensure unsupported filters throw //
//////////////////////////////////////
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
.rootCause()
.hasMessageContaining("Unable to query filesystem table by field");
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
.rootCause()
.hasMessageContaining("Unable to query filename field using operator");
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// note that we used to try unsupported filters here, expecting them to throw - but those are //
// more-or-less now implemented in the base class's query method, so, no longer expected to throw here. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
}

74
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java
View File

@ -26,9 +26,6 @@ import java.util.List;
import java.util.UUID;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@ -38,7 +35,6 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -77,53 +73,59 @@ public class S3BackendModuleTest extends BaseS3Test
/////////////////////////////////////////
// filter for a file name that's found //
/////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
files = actionBase.listFiles(table, backend, "BLOB-2.txt");
assertEquals(1, files.size());
assertThat(files.get(0).getKey()).contains("BLOB-2.txt");
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
files = actionBase.listFiles(table, backend, "BLOB-1.txt");
assertEquals(1, files.size());
assertThat(files.get(0).getKey()).contains("BLOB-1.txt");
///////////////////////////////////
// filter for 2 names that exist //
///////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
assertEquals(2, files.size());
///////////////////////////
// not supported anymore //
///////////////////////////
// ///////////////////////////////////
// // filter for 2 names that exist //
// ///////////////////////////////////
// files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
// assertEquals(2, files.size());
/////////////////////////////////////////////
// filter for a file name that isn't found //
/////////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
files = actionBase.listFiles(table, backend, "NOT-FOUND.txt");
assertEquals(0, files.size());
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
assertEquals(1, files.size());
///////////////////////////
// not supported anymore //
///////////////////////////
// files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
// assertEquals(1, files.size());
////////////////////////////////////////////////////
// 2 criteria, and'ed, and can't match, so find 0 //
////////////////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
assertEquals(0, files.size());
///////////////////////////
// not supported anymore //
///////////////////////////
// ////////////////////////////////////////////////////
// // 2 criteria, and'ed, and can't match, so find 0 //
// ////////////////////////////////////////////////////
// files = actionBase.listFiles(table, backend, new QQueryFilter(
// new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
// new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
// assertEquals(0, files.size());
//////////////////////////////////////////////////
// 2 criteria, or'ed, and both match, so find 2 //
//////////////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
.withBooleanOperator(QQueryFilter.BooleanOperator.OR));
assertEquals(2, files.size());
// //////////////////////////////////////////////////
// // 2 criteria, or'ed, and both match, so find 2 //
// //////////////////////////////////////////////////
// files = actionBase.listFiles(table, backend, new QQueryFilter(
// new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
// new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
// .withBooleanOperator(QQueryFilter.BooleanOperator.OR));
// assertEquals(2, files.size());
//////////////////////////////////////
// ensure unsupported filters throw //
//////////////////////////////////////
assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
.hasMessageContaining("Unable to query filesystem table by field");
assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
.hasMessageContaining("Unable to query filename field using operator");
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// note that we used to try unsupported filters here, expecting them to throw - but those are //
// more-or-less now implemented in the base class's query method, so, no longer expected to throw here. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
}

67
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java
View File

@ -23,15 +23,11 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -62,69 +58,6 @@ class SFTPQueryActionTest extends BaseSFTPTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testQueryWithPath() throws Exception
{
String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/";
try
{
copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt");
copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt");
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE)
.withFilter(new QQueryFilter(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
Assertions.assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows from subfolder path query");
}
finally
{
rmrfInContainer(subfolderPath);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testQueryWithPathAndNameLike() throws Exception
{
String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/";
try
{
copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt");
copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt");
copyFileToContainer("files/testfile.txt", subfolderPath + "/who.txt");
Map<String, Integer> patternExpectedCountMap = Map.of(
"%.txt", 3,
"sub%", 2,
"%1%", 1,
"%", 3,
"*", 0
);
for(Map.Entry<String, Integer> entry : patternExpectedCountMap.entrySet())
{
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE).withFilter(new QQueryFilter()
.withCriteria(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder"))
.withCriteria(new QFilterCriteria("baseName", QCriteriaOperator.LIKE, entry.getKey())));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
Assertions.assertEquals(entry.getValue(), queryOutput.getRecords().size(), "Expected # of rows from subfolder path, baseName like: " + entry.getKey());
}
}
finally
{
rmrfInContainer(subfolderPath);
}
}
/*******************************************************************************
**
*******************************************************************************/