QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-773 avoid s3 list requests that start with / if backend & table have no basePaths

Browse Source
This commit is contained in:
darin.kelkhoff
2023-12-29 19:12:34 -06:00
parent 3f431b39b9
commit 92b052fe59
4 changed files with 110 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java
View File

@ -117,7 +117,15 @@ public class S3Utils
QFilterCriteria criteria = filter.getCriteria().get(0);
if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS))
{
prefix += "/" + criteria.getValues().get(0);
if(!prefix.isEmpty())
{
///////////////////////////////////////////////////////
// remember, a prefix starting with / finds nothing! //
///////////////////////////////////////////////////////
prefix += "/";
}
prefix += criteria.getValues().get(0);
}
}
}

43
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -55,9 +55,10 @@ import org.apache.commons.io.FileUtils;
*******************************************************************************/
public class TestUtils
{
public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
public static final String BACKEND_NAME_S3 = "s3";
public static final String BACKEND_NAME_MOCK = "mock";
public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
public static final String BACKEND_NAME_S3 = "s3";
public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix";
public static final String BACKEND_NAME_MOCK = "mock";
public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json";
public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv";
@ -65,6 +66,7 @@ public class TestUtils
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
public static final String TABLE_NAME_BLOB_S3 = "s3-blob";
public static final String TABLE_NAME_PERSON_MOCK = "person-mock";
public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix";
public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed";
@ -135,8 +137,10 @@ public class TestUtils
qInstance.addTable(defineLocalFilesystemCSVPersonTable());
qInstance.addTable(defineLocalFilesystemBlobTable());
qInstance.addBackend(defineS3Backend());
qInstance.addBackend(defineS3BackendSansPrefix());
qInstance.addTable(defineS3CSVPersonTable());
qInstance.addTable(defineS3BlobTable());
qInstance.addTable(defineS3BlobSansPrefixTable());
qInstance.addBackend(defineMockBackend());
qInstance.addTable(defineMockPersonTable());
qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
@ -275,6 +279,27 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineS3BlobSansPrefixTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_BLOB_S3_SANS_PREFIX)
.withLabel("Blob S3")
.withBackendName(defineS3BackendSansPrefix().getName())
.withPrimaryKeyField("fileName")
.withField(new QFieldMetaData("fileName", QFieldType.STRING))
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
.withBackendDetails(new S3TableBackendDetails()
.withCardinality(Cardinality.ONE)
.withFileNameFieldName("fileName")
.withContentsFieldName("contents")
);
}
/*******************************************************************************
**
*******************************************************************************/
@ -288,6 +313,18 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static S3BackendMetaData defineS3BackendSansPrefix()
{
return (new S3BackendMetaData()
.withBucketName(BaseS3Test.BUCKET_NAME_FOR_SANS_PREFIX_BACKEND)
.withName(BACKEND_NAME_S3_SANS_PREFIX));
}
/*******************************************************************************
**
*******************************************************************************/

24
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.module.filesystem.s3;
import java.util.List;
import cloud.localstack.ServiceName;
import cloud.localstack.awssdkv1.TestUtils;
import cloud.localstack.docker.LocalstackDockerExtension;
@ -46,6 +47,7 @@ public class BaseS3Test extends BaseTest
public static final String TEST_FOLDER = "test-files";
public static final String SUB_FOLDER = "sub-folder";
public static final String BUCKET_NAME_FOR_SANS_PREFIX_BACKEND = "localstack-test-bucket-sans-prefix";
/*******************************************************************************
@ -65,6 +67,11 @@ public class BaseS3Test extends BaseTest
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-1.txt", "Hello, Blob");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-2.txt", "Hi, Bob");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-3.md", "# Hi, MD");
amazonS3.createBucket(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND);
amazonS3.putObject(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND, "BLOB-1.txt", "Hello, Blob");
amazonS3.putObject(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND, "BLOB-2.txt", "Hi, Bob");
amazonS3.putObject(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND, "BLOB-3.md", "# Hi, MD");
}
@ -77,16 +84,19 @@ public class BaseS3Test extends BaseTest
{
AmazonS3 amazonS3 = getAmazonS3();
if(amazonS3.doesBucketExistV2(BUCKET_NAME))
for(String bucketName : List.of(BUCKET_NAME, BUCKET_NAME_FOR_SANS_PREFIX_BACKEND))
{
////////////////////////
// todo - paginate... //
////////////////////////
for(S3ObjectSummary objectSummary : amazonS3.listObjectsV2(BUCKET_NAME).getObjectSummaries())
if(amazonS3.doesBucketExistV2(bucketName))
{
amazonS3.deleteObject(BUCKET_NAME, objectSummary.getKey());
////////////////////////
// todo - paginate... //
////////////////////////
for(S3ObjectSummary objectSummary : amazonS3.listObjectsV2(bucketName).getObjectSummaries())
{
amazonS3.deleteObject(bucketName, objectSummary.getKey());
}
amazonS3.deleteBucket(bucketName);
}
amazonS3.deleteBucket(BUCKET_NAME);
}
}

44
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java
View File

@ -114,4 +114,48 @@ public class S3QueryActionTest extends BaseS3Test
assertEquals(1, queryOutput.getRecords().size(), "Query with limit should be respected");
}
/*******************************************************************************
** We had a bug where, if both the backend and table have no basePath ("prefix"),
** then our file-listing was doing a request with a prefix starting with /, which
** causes no results, so, this test is to show that isn't happening.
*******************************************************************************/
@Test
public void testQueryForCardinalityOneInBackendWithoutPrefix() throws QException
{
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_S3_SANS_PREFIX);
queryInput.setFilter(new QQueryFilter());
S3QueryAction s3QueryAction = new S3QueryAction();
s3QueryAction.setS3Utils(getS3Utils());
QueryOutput queryOutput = s3QueryAction.execute(queryInput);
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
queryOutput = s3QueryAction.execute(queryInput);
assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
////////////////////////////////////////////////////////////////
// put a glob on the table - now should only find 2 txt files //
////////////////////////////////////////////////////////////////
QInstance instance = TestUtils.defineInstance();
((S3TableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_S3_SANS_PREFIX).getBackendDetails()))
.withGlob("*.txt");
reInitInstanceInContext(instance);
queryInput.setFilter(new QQueryFilter());
queryOutput = s3QueryAction.execute(queryInput);
assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
//////////////////////////////
// add a limit to the query //
//////////////////////////////
queryInput.setFilter(new QQueryFilter().withLimit(1));
queryOutput = s3QueryAction.execute(queryInput);
assertEquals(1, queryOutput.getRecords().size(), "Query with limit should be respected");
}
}