QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-19 05:30:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-773 update to support listing/filtering filesystem tables with Cardinality.ONE (single-record per-file)

Browse Source
This commit is contained in:
darin.kelkhoff
2023-12-27 16:10:45 -06:00
parent 1d022200c5
commit 940080bc86
13 changed files with 655 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java
View File

@ -25,6 +25,11 @@ package com.kingsrook.qqq.backend.module.filesystem.local;
import java.io.File;
import java.io.IOException;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -36,6 +41,8 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
@ -47,6 +54,9 @@ public class FilesystemBackendModuleTest
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
public void beforeEach() throws IOException
{
@ -55,6 +65,9 @@ public class FilesystemBackendModuleTest
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
public void afterEach() throws Exception
{
@ -63,6 +76,78 @@ public class FilesystemBackendModuleTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testListFiles() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
AbstractFilesystemAction abstractFilesystemAction = new AbstractFilesystemAction();
QBackendMetaData backend = qInstance.getBackendForTable(table.getName());
//////////////////////////////////////////////////////////
// with no filter given, all (3) files should come back //
//////////////////////////////////////////////////////////
List<File> files = abstractFilesystemAction.listFiles(table, backend);
assertEquals(3, files.size());
/////////////////////////////////////////
// filter for a file name that's found //
/////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
assertEquals(1, files.size());
assertEquals("BLOB-2.txt", files.get(0).getName());
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
assertEquals(1, files.size());
assertEquals("BLOB-1.txt", files.get(0).getName());
///////////////////////////////////
// filter for 2 names that exist //
///////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
assertEquals(2, files.size());
/////////////////////////////////////////////
// filter for a file name that isn't found //
/////////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
assertEquals(0, files.size());
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
assertEquals(1, files.size());
////////////////////////////////////////////////////
// 2 criteria, and'ed, and can't match, so find 0 //
////////////////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
assertEquals(0, files.size());
//////////////////////////////////////////////////
// 2 criteria, or'ed, and both match, so find 2 //
//////////////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
.withBooleanOperator(QQueryFilter.BooleanOperator.OR));
assertEquals(2, files.size());
//////////////////////////////////////
// ensure unsupported filters throw //
//////////////////////////////////////
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
.hasMessageContaining("Unable to query filesystem table by field");
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
.hasMessageContaining("Unable to query filename field using operator");
}
/*******************************************************************************
**
*******************************************************************************/

32
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java
View File

@ -89,6 +89,7 @@ public class FilesystemActionTest extends BaseTest
writePersonJSONFiles(baseDirectory);
writePersonCSVFiles(baseDirectory);
writeBlobFiles(baseDirectory);
}
@ -158,6 +159,37 @@ public class FilesystemActionTest extends BaseTest
/*******************************************************************************
** Write some data files into the directory for the filesystem module.
*******************************************************************************/
private void writeBlobFiles(File baseDirectory) throws IOException
{
String fullPath = baseDirectory.getAbsolutePath();
if(TestUtils.defineLocalFilesystemBlobTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
{
if(StringUtils.hasContent(details.getBasePath()))
{
fullPath += File.separatorChar + details.getBasePath();
}
}
fullPath += File.separatorChar;
String data1 = """
Hello, Blob
""";
FileUtils.writeStringToFile(new File(fullPath + "BLOB-1.txt"), data1);
String data2 = """
Hi Bob""";
FileUtils.writeStringToFile(new File(fullPath + "BLOB-2.txt"), data2);
String data3 = """
# Hi MD...""";
FileUtils.writeStringToFile(new File(fullPath + "BLOB-3.md"), data3);
}
/*******************************************************************************
**
*******************************************************************************/

49
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java
View File

@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -32,8 +35,10 @@ import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractPostReadFileCustomizer;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.FilesystemTableCustomizers;
import org.junit.jupiter.api.Assertions;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
@ -51,8 +56,8 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
QueryInput queryInput = new QueryInput();
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
Assertions.assertTrue(queryOutput.getRecords().stream()
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
assertTrue(queryOutput.getRecords().stream()
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
"All records should have a full-path in their backend details, matching the test folder name");
}
@ -74,14 +79,48 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
Assertions.assertTrue(
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
assertTrue(
queryOutput.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")),
"All records should have their email addresses up-shifted.");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQueryForCardinalityOne() throws QException
{
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
queryInput.setFilter(new QQueryFilter());
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
queryOutput = new FilesystemQueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
////////////////////////////////////////////////////////////////
// put a glob on the table - now should only find 2 txt files //
////////////////////////////////////////////////////////////////
QInstance instance = TestUtils.defineInstance();
((FilesystemTableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).getBackendDetails()))
.withGlob("*.txt");
reInitInstanceInContext(instance);
queryInput.setFilter(new QQueryFilter());
queryOutput = new FilesystemQueryAction().execute(queryInput);
assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
}
/*******************************************************************************
**
*******************************************************************************/
public static class ValueUpshifter extends AbstractPostReadFileCustomizer
{
@Override

6
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java
View File

@ -26,7 +26,7 @@ import java.util.List;
import java.util.stream.Collectors;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@ -187,7 +187,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
/*******************************************************************************
**
*******************************************************************************/
private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException
private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QException
{
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
@ -207,7 +207,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
/*******************************************************************************
**
*******************************************************************************/
private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException
private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QException
{
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();

3
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java
View File

@ -62,6 +62,9 @@ public class BaseS3Test extends BaseTest
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-1.txt", "Hello, Blob");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-2.txt", "Hi, Bob");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-3.md", "# Hi, MD");
}

85
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java
View File

@ -25,6 +25,11 @@ package com.kingsrook.qqq.backend.module.filesystem.s3;
import java.util.List;
import java.util.UUID;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -32,6 +37,9 @@ import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemExceptio
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
@ -43,6 +51,83 @@ public class S3BackendModuleTest extends BaseS3Test
/*******************************************************************************
**
*******************************************************************************/
@Test
void testListFiles() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_S3);
QBackendMetaData backend = qInstance.getBackendForTable(table.getName());
//////////////////////////////////////////////////////
// set up the backend module (e.g., for localstack) //
//////////////////////////////////////////////////////
S3BackendModule s3BackendModule = new S3BackendModule();
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
actionBase.setS3Utils(getS3Utils());
//////////////////////////////////////////////////////////
// with no filter given, all (3) files should come back //
//////////////////////////////////////////////////////////
List<S3ObjectSummary> files = actionBase.listFiles(table, backend);
assertEquals(3, files.size());
/////////////////////////////////////////
// filter for a file name that's found //
/////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
assertEquals(1, files.size());
assertThat(files.get(0).getKey()).contains("BLOB-2.txt");
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
assertEquals(1, files.size());
assertThat(files.get(0).getKey()).contains("BLOB-1.txt");
///////////////////////////////////
// filter for 2 names that exist //
///////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
assertEquals(2, files.size());
/////////////////////////////////////////////
// filter for a file name that isn't found //
/////////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
assertEquals(0, files.size());
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
assertEquals(1, files.size());
////////////////////////////////////////////////////
// 2 criteria, and'ed, and can't match, so find 0 //
////////////////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
assertEquals(0, files.size());
//////////////////////////////////////////////////
// 2 criteria, or'ed, and both match, so find 2 //
//////////////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
.withBooleanOperator(QQueryFilter.BooleanOperator.OR));
assertEquals(2, files.size());
//////////////////////////////////////
// ensure unsupported filters throw //
//////////////////////////////////////
assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
.hasMessageContaining("Unable to query filesystem table by field");
assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
.hasMessageContaining("Unable to query filename field using operator");
}
/*******************************************************************************
**
*******************************************************************************/

41
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java
View File

@ -23,13 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
@ -66,4 +72,39 @@ public class S3QueryActionTest extends BaseS3Test
return queryInput;
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQueryForCardinalityOne() throws QException
{
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_S3);
queryInput.setFilter(new QQueryFilter());
S3QueryAction s3QueryAction = new S3QueryAction();
s3QueryAction.setS3Utils(getS3Utils());
QueryOutput queryOutput = s3QueryAction.execute(queryInput);
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
queryOutput = s3QueryAction.execute(queryInput);
assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
////////////////////////////////////////////////////////////////
// put a glob on the table - now should only find 2 txt files //
////////////////////////////////////////////////////////////////
QInstance instance = TestUtils.defineInstance();
((S3TableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_S3).getBackendDetails()))
.withGlob("*.txt");
reInitInstanceInContext(instance);
queryInput.setFilter(new QQueryFilter());
queryOutput = s3QueryAction.execute(queryInput);
assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
}
}

12
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java
View File

@ -22,10 +22,10 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Test;
@ -42,14 +42,14 @@ public class S3UtilsTest extends BaseS3Test
**
*******************************************************************************/
@Test
public void testListObjectsInBucketAtPath()
public void testListObjectsInBucketAtPath() throws QException
{
S3Utils s3Utils = getS3Utils();
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders");
assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders");
assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders");
assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
assertEquals(7, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes");
@ -60,8 +60,8 @@ public class S3UtilsTest extends BaseS3Test
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string");
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
}
@ -70,7 +70,7 @@ public class S3UtilsTest extends BaseS3Test
**
*******************************************************************************/
@Test
public void testGetObjectAsInputStream() throws IOException
public void testGetObjectAsInputStream() throws Exception
{
S3Utils s3Utils = getS3Utils();
List<S3ObjectSummary> s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", "");