diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java index 2a22dbe9..a49ca3de 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java @@ -117,7 +117,15 @@ public class S3Utils QFilterCriteria criteria = filter.getCriteria().get(0); if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS)) { - prefix += "/" + criteria.getValues().get(0); + if(!prefix.isEmpty()) + { + /////////////////////////////////////////////////////// + // remember, a prefix starting with / finds nothing! // + /////////////////////////////////////////////////////// + prefix += "/"; + } + + prefix += criteria.getValues().get(0); } } } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index 4510ecd5..cab98928 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -55,9 +55,10 @@ import org.apache.commons.io.FileUtils; *******************************************************************************/ public class TestUtils { - public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; - public static final String BACKEND_NAME_S3 = "s3"; - public static final String BACKEND_NAME_MOCK = "mock"; + public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; + public static final String BACKEND_NAME_S3 = "s3"; + public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; + public static final String BACKEND_NAME_MOCK = "mock"; public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json"; public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv"; @@ -65,6 +66,7 @@ public class TestUtils public static final String TABLE_NAME_PERSON_S3 = "person-s3"; public static final String TABLE_NAME_BLOB_S3 = "s3-blob"; public static final String TABLE_NAME_PERSON_MOCK = "person-mock"; + public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix"; public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed"; @@ -135,8 +137,10 @@ public class TestUtils qInstance.addTable(defineLocalFilesystemCSVPersonTable()); qInstance.addTable(defineLocalFilesystemBlobTable()); qInstance.addBackend(defineS3Backend()); + qInstance.addBackend(defineS3BackendSansPrefix()); qInstance.addTable(defineS3CSVPersonTable()); qInstance.addTable(defineS3BlobTable()); + qInstance.addTable(defineS3BlobSansPrefixTable()); qInstance.addBackend(defineMockBackend()); qInstance.addTable(defineMockPersonTable()); qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess()); @@ -275,6 +279,27 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineS3BlobSansPrefixTable() + { + return new QTableMetaData() + .withName(TABLE_NAME_BLOB_S3_SANS_PREFIX) + .withLabel("Blob S3") + .withBackendName(defineS3BackendSansPrefix().getName()) + .withPrimaryKeyField("fileName") + .withField(new QFieldMetaData("fileName", QFieldType.STRING)) + .withField(new QFieldMetaData("contents", QFieldType.BLOB)) + .withBackendDetails(new S3TableBackendDetails() + .withCardinality(Cardinality.ONE) + .withFileNameFieldName("fileName") + .withContentsFieldName("contents") + ); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -288,6 +313,18 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static S3BackendMetaData defineS3BackendSansPrefix() + { + return (new S3BackendMetaData() + .withBucketName(BaseS3Test.BUCKET_NAME_FOR_SANS_PREFIX_BACKEND) + .withName(BACKEND_NAME_S3_SANS_PREFIX)); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java index e9538d4b..9cf02b29 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3; +import java.util.List; import cloud.localstack.ServiceName; import cloud.localstack.awssdkv1.TestUtils; import cloud.localstack.docker.LocalstackDockerExtension; @@ -46,6 +47,7 @@ public class BaseS3Test extends BaseTest public static final String TEST_FOLDER = "test-files"; public static final String SUB_FOLDER = "sub-folder"; + public static final String BUCKET_NAME_FOR_SANS_PREFIX_BACKEND = "localstack-test-bucket-sans-prefix"; /******************************************************************************* @@ -65,6 +67,11 @@ public class BaseS3Test extends BaseTest amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-1.txt", "Hello, Blob"); amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-2.txt", "Hi, Bob"); amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-3.md", "# Hi, MD"); + + amazonS3.createBucket(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND); + amazonS3.putObject(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND, "BLOB-1.txt", "Hello, Blob"); + amazonS3.putObject(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND, "BLOB-2.txt", "Hi, Bob"); + amazonS3.putObject(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND, "BLOB-3.md", "# Hi, MD"); } @@ -77,16 +84,19 @@ public class BaseS3Test extends BaseTest { AmazonS3 amazonS3 = getAmazonS3(); - if(amazonS3.doesBucketExistV2(BUCKET_NAME)) + for(String bucketName : List.of(BUCKET_NAME, BUCKET_NAME_FOR_SANS_PREFIX_BACKEND)) { - //////////////////////// - // todo - paginate... // - //////////////////////// - for(S3ObjectSummary objectSummary : amazonS3.listObjectsV2(BUCKET_NAME).getObjectSummaries()) + if(amazonS3.doesBucketExistV2(bucketName)) { - amazonS3.deleteObject(BUCKET_NAME, objectSummary.getKey()); + //////////////////////// + // todo - paginate... // + //////////////////////// + for(S3ObjectSummary objectSummary : amazonS3.listObjectsV2(bucketName).getObjectSummaries()) + { + amazonS3.deleteObject(bucketName, objectSummary.getKey()); + } + amazonS3.deleteBucket(bucketName); } - amazonS3.deleteBucket(BUCKET_NAME); } } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java index 4e97044a..7e07b58d 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java @@ -114,4 +114,48 @@ public class S3QueryActionTest extends BaseS3Test assertEquals(1, queryOutput.getRecords().size(), "Query with limit should be respected"); } + + + /******************************************************************************* + ** We had a bug where, if both the backend and table have no basePath ("prefix"), + ** then our file-listing was doing a request with a prefix starting with /, which + ** causes no results, so, this test is to show that isn't happening. + *******************************************************************************/ + @Test + public void testQueryForCardinalityOneInBackendWithoutPrefix() throws QException + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_S3_SANS_PREFIX); + queryInput.setFilter(new QQueryFilter()); + + S3QueryAction s3QueryAction = new S3QueryAction(); + s3QueryAction.setS3Utils(getS3Utils()); + + QueryOutput queryOutput = s3QueryAction.execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); + queryOutput = s3QueryAction.execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row"); + assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName")); + + //////////////////////////////////////////////////////////////// + // put a glob on the table - now should only find 2 txt files // + //////////////////////////////////////////////////////////////// + QInstance instance = TestUtils.defineInstance(); + ((S3TableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_S3_SANS_PREFIX).getBackendDetails())) + .withGlob("*.txt"); + reInitInstanceInContext(instance); + + queryInput.setFilter(new QQueryFilter()); + queryOutput = s3QueryAction.execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows"); + + ////////////////////////////// + // add a limit to the query // + ////////////////////////////// + queryInput.setFilter(new QQueryFilter().withLimit(1)); + queryOutput = s3QueryAction.execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Query with limit should be respected"); + } + } \ No newline at end of file