QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-773 update to support listing/filtering filesystem tables with Cardinality.ONE (single-record per-file)

Browse Source
This commit is contained in:
darin.kelkhoff
2023-12-27 16:10:45 -06:00
parent 1d022200c5
commit 940080bc86
13 changed files with 655 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
View File

@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -68,7 +69,17 @@ public abstract class AbstractBaseFilesystemAction<FILE>
/*******************************************************************************
** List the files for a table - to be implemented in module-specific subclasses.
*******************************************************************************/
public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase);
public List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase) throws QException
{
return (listFiles(table, backendBase, null));
}
/*******************************************************************************
** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses.
*******************************************************************************/
public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException;
/*******************************************************************************
** Read the contents of a file - to be implemented in module-specific subclasses.
@ -181,6 +192,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
/*******************************************************************************
** Generic implementation of the execute method from the QueryInterface
*******************************************************************************/
@SuppressWarnings("checkstyle:Indentation")
public QueryOutput executeQuery(QueryInput queryInput) throws QException
{
preAction(queryInput.getBackend());
@ -191,51 +203,76 @@ public abstract class AbstractBaseFilesystemAction<FILE>
QTableMetaData table = queryInput.getTable();
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
List<FILE> files = listFiles(table, queryInput.getBackend());
List<FILE> files = listFiles(table, queryInput.getBackend(), queryInput.getFilter());
for(FILE file : files)
{
LOG.info("Processing file: " + getFullPathForFile(file));
switch(tableDetails.getRecordFormat())
{
case CSV:
{
String fileContents = IOUtils.toString(readFile(file));
fileContents = customizeFileContentsAfterReading(table, fileContents);
if(queryInput.getRecordPipe() != null)
InputStream inputStream = readFile(file);
switch(tableDetails.getCardinality())
{
case MANY:
{
switch(tableDetails.getRecordFormat())
{
new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record ->
case CSV:
{
////////////////////////////////////////////////////////////////////////////////////////////
// Before the records go into the pipe, make sure their backend details are added to them //
////////////////////////////////////////////////////////////////////////////////////////////
addBackendDetailsToRecord(record, file);
}));
}
else
{
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
queryOutput.addRecords(recordsInFile);
String fileContents = IOUtils.toString(inputStream);
fileContents = customizeFileContentsAfterReading(table, fileContents);
if(queryInput.getRecordPipe() != null)
{
new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record ->
{
////////////////////////////////////////////////////////////////////////////////////////////
// Before the records go into the pipe, make sure their backend details are added to them //
////////////////////////////////////////////////////////////////////////////////////////////
addBackendDetailsToRecord(record, file);
}));
}
else
{
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
queryOutput.addRecords(recordsInFile);
}
break;
}
case JSON:
{
String fileContents = IOUtils.toString(inputStream);
fileContents = customizeFileContentsAfterReading(table, fileContents);
// todo - pipe support!!
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
queryOutput.addRecords(recordsInFile);
break;
}
default:
{
throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat());
}
}
break;
}
case JSON:
case ONE:
{
String fileContents = IOUtils.toString(readFile(file));
fileContents = customizeFileContentsAfterReading(table, fileContents);
String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table);
// todo - pipe support!!
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
byte[] bytes = inputStream.readAllBytes();
QRecord record = new QRecord()
.withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase)
.withValue(tableDetails.getContentsFieldName(), bytes);
queryOutput.addRecord(record);
queryOutput.addRecords(recordsInFile);
break;
}
default:
{
throw new NotImplementedException("Filesystem record format " + tableDetails.getRecordFormat() + " is not yet implemented");
throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality());
}
}
}
@ -342,8 +379,8 @@ public abstract class AbstractBaseFilesystemAction<FILE>
{
for(QRecord record : insertInput.getRecords())
{
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString("fileName"));
writeFile(backend, fullPath, record.getValueByteArray("contents"));
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
output.addRecord(record);
}

68
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java
View File

@ -35,6 +35,11 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
private RecordFormat recordFormat;
private Cardinality cardinality;
///////////////////////////////////////////////////////////////////////////////////////////////////
// todo default these to null, and give validation error if not set for a cardinality=ONE table? //
///////////////////////////////////////////////////////////////////////////////////////////////////
private String contentsFieldName = "contents";
private String fileNameFieldName = "fileName";
/*******************************************************************************
@ -175,4 +180,67 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
return ((T) this);
}
/*******************************************************************************
** Getter for contentsFieldName
*******************************************************************************/
public String getContentsFieldName()
{
return (this.contentsFieldName);
}
/*******************************************************************************
** Setter for contentsFieldName
*******************************************************************************/
public void setContentsFieldName(String contentsFieldName)
{
this.contentsFieldName = contentsFieldName;
}
/*******************************************************************************
** Fluent setter for contentsFieldName
*******************************************************************************/
public AbstractFilesystemTableBackendDetails withContentsFieldName(String contentsFieldName)
{
this.contentsFieldName = contentsFieldName;
return (this);
}
/*******************************************************************************
** Getter for fileNameFieldName
*******************************************************************************/
public String getFileNameFieldName()
{
return (this.fileNameFieldName);
}
/*******************************************************************************
** Setter for fileNameFieldName
*******************************************************************************/
public void setFileNameFieldName(String fileNameFieldName)
{
this.fileNameFieldName = fileNameFieldName;
}
/*******************************************************************************
** Fluent setter for fileNameFieldName
*******************************************************************************/
public AbstractFilesystemTableBackendDetails withFileNameFieldName(String fileNameFieldName)
{
this.fileNameFieldName = fileNameFieldName;
return (this);
}
}

77
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java
View File

@ -23,19 +23,36 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOCase;
import org.apache.commons.io.filefilter.AndFileFilter;
import org.apache.commons.io.filefilter.NameFileFilter;
import org.apache.commons.io.filefilter.OrFileFilter;
import org.apache.commons.io.filefilter.TrueFileFilter;
import org.apache.commons.io.filefilter.WildcardFileFilter;
/*******************************************************************************
@ -51,12 +68,66 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
** List the files for this table.
*******************************************************************************/
@Override
public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase)
public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
{
// todo - needs rewritten to do globbing...
String fullPath = getFullBasePath(table, backendBase);
File directory = new File(fullPath);
File[] files = directory.listFiles();
File[] files = null;
AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
FileFilter fileFilter = TrueFileFilter.INSTANCE;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if each file is its own record (ONE), then we may need to do filtering of the directory listing based on the input filter //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(Cardinality.ONE.equals(tableBackendDetails.getCardinality()))
{
if(filter != null && filter.hasAnyCriteria())
{
List<FileFilter> fileFilterList = new ArrayList<>();
for(QFilterCriteria criteria : filter.getCriteria())
{
if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
{
if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
{
fileFilterList.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(0))));
}
else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
{
List<NameFileFilter> nameInFilters = new ArrayList<>();
for(int i = 0; i < criteria.getValues().size(); i++)
{
nameInFilters.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(i))));
}
fileFilterList.add(new OrFileFilter(nameInFilters));
}
else
{
throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
}
}
else
{
throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
}
}
fileFilter = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? new AndFileFilter(fileFilterList) : new OrFileFilter(fileFilterList);
}
}
///////////////////////////////////////////////////////////////////////////////////////////
// if the table has a glob specified, add it as an AND to the filter built to this point //
///////////////////////////////////////////////////////////////////////////////////////////
if(StringUtils.hasContent(tableBackendDetails.getGlob()))
{
WildcardFileFilter globFilenameFilter = new WildcardFileFilter(tableBackendDetails.getGlob(), IOCase.INSENSITIVE);
fileFilter = new AndFileFilter(List.of(globFilenameFilter, fileFilter));
}
files = directory.listFiles(fileFilter);
if(files == null)
{

6
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java
View File

@ -30,7 +30,9 @@ import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@ -126,7 +128,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
** List the files for a table.
*******************************************************************************/
@Override
public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase)
public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
{
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase);
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
@ -138,7 +140,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
////////////////////////////////////////////////////////////////////
// todo - look at metadata to configure the s3 client here? //
////////////////////////////////////////////////////////////////////
return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob);
return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob, filter, tableDetails);
}

144
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java
View File

@ -37,7 +37,14 @@ import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
@ -61,7 +68,19 @@ public class S3Utils
** List the objects in an S3 bucket matching a glob, per:
** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
*******************************************************************************/
public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob)
public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob) throws QException
{
return listObjectsInBucketMatchingGlob(bucketName, path, glob, null, null);
}
/*******************************************************************************
** List the objects in an S3 bucket matching a glob, per:
** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
** and also - (possibly) apply a file-name filter (based on the table's details).
*******************************************************************************/
public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
{
//////////////////////////////////////////////////////////////////////////////////////////////////
// s3 list requests find nothing if the path starts with a /, so strip away any leading slashes //
@ -77,6 +96,28 @@ public class S3Utils
prefix = prefix.substring(0, prefix.indexOf('*'));
}
///////////////////////////////////////////////////////////////////////////////////////////////////////
// for a file-per-record (ONE) table, we may need to apply the filter to listing. //
// but for MANY tables, the filtering would be done on the records after they came out of the files. //
///////////////////////////////////////////////////////////////////////////////////////////////////////
boolean useQQueryFilter = false;
if(tableDetails != null && Cardinality.ONE.equals(tableDetails.getCardinality()))
{
useQQueryFilter = true;
}
if(filter != null && useQQueryFilter)
{
if(filter.getCriteria() != null && filter.getCriteria().size() == 1)
{
QFilterCriteria criteria = filter.getCriteria().get(0);
if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS))
{
prefix += "/" + criteria.getValues().get(0);
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob. //
// That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. //
@ -86,6 +127,7 @@ public class S3Utils
{
glob = "";
}
if(!glob.contains("*"))
{
if(glob.equals(""))
@ -114,7 +156,7 @@ public class S3Utils
{
listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken());
}
LOG.info("Listing bucket=" + bucketName + ", path=" + path);
LOG.info("Listing bucket=" + bucketName + ", path=" + path + ", prefix=" + prefix + ", glob=" + glob);
listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request);
//////////////////////////////////
@ -149,6 +191,14 @@ public class S3Utils
continue;
}
///////////////////////////////////////////////////////////////////////////////////
// if we're a file-per-record table, and we have a filter, compare the key to it //
///////////////////////////////////////////////////////////////////////////////////
if(!doesObjectKeyMatchFilter(key, filter, tableDetails))
{
continue;
}
rs.add(objectSummary);
}
}
@ -159,6 +209,95 @@ public class S3Utils
/*******************************************************************************
**
*******************************************************************************/
private boolean doesObjectKeyMatchFilter(String key, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
{
if(filter == null || !filter.hasAnyCriteria())
{
return (true);
}
Path path = Path.of(URI.create("file:///" + key));
////////////////////////////////////////////////////////////////////////////////////////////////////
// foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches //
////////////////////////////////////////////////////////////////////////////////////////////////////
for(QFilterCriteria criteria : filter.getCriteria())
{
boolean matches = doesObjectKeyMatchOneCriteria(criteria, tableDetails, path);
if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
{
////////////////////////////////////////////////////////////////////////////////
// if it's not a match, and it's an AND filter, then the whole thing is false //
////////////////////////////////////////////////////////////////////////////////
return (false);
}
if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
{
////////////////////////////////////////////////////////////
// if it's an OR filter, and we've a match, return a true //
////////////////////////////////////////////////////////////
return (true);
}
}
//////////////////////////////////////////////////////////////////////
// if we didn't return above, return now //
// for an OR - if we didn't find something true, then return false. //
// else, an AND - if we didn't find a false, we can return true. //
//////////////////////////////////////////////////////////////////////
if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
{
return (false);
}
return (true);
}
/*******************************************************************************
**
*******************************************************************************/
private static boolean doesObjectKeyMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException
{
if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
{
if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
{
return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path));
}
else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
{
boolean anyMatch = false;
for(int i = 0; i < criteria.getValues().size(); i++)
{
if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path))
{
anyMatch = true;
break;
}
}
return (anyMatch);
}
else
{
throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
}
}
else
{
throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
}
}
/*******************************************************************************
** Get the contents (as an InputStream) for an object in s3
*******************************************************************************/
@ -245,4 +384,5 @@ public class S3Utils
return amazonS3;
}
}

85
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java
View File

@ -25,6 +25,11 @@ package com.kingsrook.qqq.backend.module.filesystem.local;
import java.io.File;
import java.io.IOException;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -36,6 +41,8 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
@ -47,6 +54,9 @@ public class FilesystemBackendModuleTest
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
public void beforeEach() throws IOException
{
@ -55,6 +65,9 @@ public class FilesystemBackendModuleTest
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
public void afterEach() throws Exception
{
@ -63,6 +76,78 @@ public class FilesystemBackendModuleTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testListFiles() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
AbstractFilesystemAction abstractFilesystemAction = new AbstractFilesystemAction();
QBackendMetaData backend = qInstance.getBackendForTable(table.getName());
//////////////////////////////////////////////////////////
// with no filter given, all (3) files should come back //
//////////////////////////////////////////////////////////
List<File> files = abstractFilesystemAction.listFiles(table, backend);
assertEquals(3, files.size());
/////////////////////////////////////////
// filter for a file name that's found //
/////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
assertEquals(1, files.size());
assertEquals("BLOB-2.txt", files.get(0).getName());
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
assertEquals(1, files.size());
assertEquals("BLOB-1.txt", files.get(0).getName());
///////////////////////////////////
// filter for 2 names that exist //
///////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
assertEquals(2, files.size());
/////////////////////////////////////////////
// filter for a file name that isn't found //
/////////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
assertEquals(0, files.size());
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
assertEquals(1, files.size());
////////////////////////////////////////////////////
// 2 criteria, and'ed, and can't match, so find 0 //
////////////////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
assertEquals(0, files.size());
//////////////////////////////////////////////////
// 2 criteria, or'ed, and both match, so find 2 //
//////////////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
.withBooleanOperator(QQueryFilter.BooleanOperator.OR));
assertEquals(2, files.size());
//////////////////////////////////////
// ensure unsupported filters throw //
//////////////////////////////////////
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
.hasMessageContaining("Unable to query filesystem table by field");
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
.hasMessageContaining("Unable to query filename field using operator");
}
/*******************************************************************************
**
*******************************************************************************/

32
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java
View File

@ -89,6 +89,7 @@ public class FilesystemActionTest extends BaseTest
writePersonJSONFiles(baseDirectory);
writePersonCSVFiles(baseDirectory);
writeBlobFiles(baseDirectory);
}
@ -158,6 +159,37 @@ public class FilesystemActionTest extends BaseTest
/*******************************************************************************
** Write some data files into the directory for the filesystem module.
*******************************************************************************/
private void writeBlobFiles(File baseDirectory) throws IOException
{
String fullPath = baseDirectory.getAbsolutePath();
if(TestUtils.defineLocalFilesystemBlobTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
{
if(StringUtils.hasContent(details.getBasePath()))
{
fullPath += File.separatorChar + details.getBasePath();
}
}
fullPath += File.separatorChar;
String data1 = """
Hello, Blob
""";
FileUtils.writeStringToFile(new File(fullPath + "BLOB-1.txt"), data1);
String data2 = """
Hi Bob""";
FileUtils.writeStringToFile(new File(fullPath + "BLOB-2.txt"), data2);
String data3 = """
# Hi MD...""";
FileUtils.writeStringToFile(new File(fullPath + "BLOB-3.md"), data3);
}
/*******************************************************************************
**
*******************************************************************************/

49
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java
View File

@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -32,8 +35,10 @@ import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractPostReadFileCustomizer;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.FilesystemTableCustomizers;
import org.junit.jupiter.api.Assertions;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
@ -51,8 +56,8 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
QueryInput queryInput = new QueryInput();
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
Assertions.assertTrue(queryOutput.getRecords().stream()
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
assertTrue(queryOutput.getRecords().stream()
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
"All records should have a full-path in their backend details, matching the test folder name");
}
@ -74,14 +79,48 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
Assertions.assertTrue(
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
assertTrue(
queryOutput.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")),
"All records should have their email addresses up-shifted.");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQueryForCardinalityOne() throws QException
{
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
queryInput.setFilter(new QQueryFilter());
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
queryOutput = new FilesystemQueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
////////////////////////////////////////////////////////////////
// put a glob on the table - now should only find 2 txt files //
////////////////////////////////////////////////////////////////
QInstance instance = TestUtils.defineInstance();
((FilesystemTableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).getBackendDetails()))
.withGlob("*.txt");
reInitInstanceInContext(instance);
queryInput.setFilter(new QQueryFilter());
queryOutput = new FilesystemQueryAction().execute(queryInput);
assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
}
/*******************************************************************************
**
*******************************************************************************/
public static class ValueUpshifter extends AbstractPostReadFileCustomizer
{
@Override

6
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java
View File

@ -26,7 +26,7 @@ import java.util.List;
import java.util.stream.Collectors;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@ -187,7 +187,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
/*******************************************************************************
**
*******************************************************************************/
private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException
private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QException
{
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
@ -207,7 +207,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
/*******************************************************************************
**
*******************************************************************************/
private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException
private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QException
{
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();

3
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java
View File

@ -62,6 +62,9 @@ public class BaseS3Test extends BaseTest
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-1.txt", "Hello, Blob");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-2.txt", "Hi, Bob");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-3.md", "# Hi, MD");
}

85
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java
View File

@ -25,6 +25,11 @@ package com.kingsrook.qqq.backend.module.filesystem.s3;
import java.util.List;
import java.util.UUID;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -32,6 +37,9 @@ import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemExceptio
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
@ -43,6 +51,83 @@ public class S3BackendModuleTest extends BaseS3Test
/*******************************************************************************
**
*******************************************************************************/
@Test
void testListFiles() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_S3);
QBackendMetaData backend = qInstance.getBackendForTable(table.getName());
//////////////////////////////////////////////////////
// set up the backend module (e.g., for localstack) //
//////////////////////////////////////////////////////
S3BackendModule s3BackendModule = new S3BackendModule();
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
actionBase.setS3Utils(getS3Utils());
//////////////////////////////////////////////////////////
// with no filter given, all (3) files should come back //
//////////////////////////////////////////////////////////
List<S3ObjectSummary> files = actionBase.listFiles(table, backend);
assertEquals(3, files.size());
/////////////////////////////////////////
// filter for a file name that's found //
/////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
assertEquals(1, files.size());
assertThat(files.get(0).getKey()).contains("BLOB-2.txt");
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
assertEquals(1, files.size());
assertThat(files.get(0).getKey()).contains("BLOB-1.txt");
///////////////////////////////////
// filter for 2 names that exist //
///////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
assertEquals(2, files.size());
/////////////////////////////////////////////
// filter for a file name that isn't found //
/////////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
assertEquals(0, files.size());
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
assertEquals(1, files.size());
////////////////////////////////////////////////////
// 2 criteria, and'ed, and can't match, so find 0 //
////////////////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
assertEquals(0, files.size());
//////////////////////////////////////////////////
// 2 criteria, or'ed, and both match, so find 2 //
//////////////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
.withBooleanOperator(QQueryFilter.BooleanOperator.OR));
assertEquals(2, files.size());
//////////////////////////////////////
// ensure unsupported filters throw //
//////////////////////////////////////
assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
.hasMessageContaining("Unable to query filesystem table by field");
assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
.hasMessageContaining("Unable to query filename field using operator");
}
/*******************************************************************************
**
*******************************************************************************/

41
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java
View File

@ -23,13 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
@ -66,4 +72,39 @@ public class S3QueryActionTest extends BaseS3Test
return queryInput;
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQueryForCardinalityOne() throws QException
{
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_S3);
queryInput.setFilter(new QQueryFilter());
S3QueryAction s3QueryAction = new S3QueryAction();
s3QueryAction.setS3Utils(getS3Utils());
QueryOutput queryOutput = s3QueryAction.execute(queryInput);
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
queryOutput = s3QueryAction.execute(queryInput);
assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
////////////////////////////////////////////////////////////////
// put a glob on the table - now should only find 2 txt files //
////////////////////////////////////////////////////////////////
QInstance instance = TestUtils.defineInstance();
((S3TableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_S3).getBackendDetails()))
.withGlob("*.txt");
reInitInstanceInContext(instance);
queryInput.setFilter(new QQueryFilter());
queryOutput = s3QueryAction.execute(queryInput);
assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
}
}

12
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java
View File

@ -22,10 +22,10 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Test;
@ -42,14 +42,14 @@ public class S3UtilsTest extends BaseS3Test
**
*******************************************************************************/
@Test
public void testListObjectsInBucketAtPath()
public void testListObjectsInBucketAtPath() throws QException
{
S3Utils s3Utils = getS3Utils();
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders");
assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders");
assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders");
assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
assertEquals(7, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes");
@ -60,8 +60,8 @@ public class S3UtilsTest extends BaseS3Test
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string");
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
}
@ -70,7 +70,7 @@ public class S3UtilsTest extends BaseS3Test
**
*******************************************************************************/
@Test
public void testGetObjectAsInputStream() throws IOException
public void testGetObjectAsInputStream() throws Exception
{
S3Utils s3Utils = getS3Utils();
List<S3ObjectSummary> s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", "");