From 940080bc865c33b1de4196bb5d7a060e8207b52f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Dec 2023 16:10:45 -0600 Subject: [PATCH] CE-773 update to support listing/filtering filesystem tables with Cardinality.ONE (single-record per-file) --- .../actions/AbstractBaseFilesystemAction.java | 99 ++++++++---- ...AbstractFilesystemTableBackendDetails.java | 68 +++++++++ .../actions/AbstractFilesystemAction.java | 77 +++++++++- .../s3/actions/AbstractS3Action.java | 6 +- .../module/filesystem/s3/utils/S3Utils.java | 144 +++++++++++++++++- .../local/FilesystemBackendModuleTest.java | 85 +++++++++++ .../local/actions/FilesystemActionTest.java | 32 ++++ .../actions/FilesystemQueryActionTest.java | 49 +++++- .../sync/FilesystemSyncProcessS3Test.java | 6 +- .../module/filesystem/s3/BaseS3Test.java | 3 + .../filesystem/s3/S3BackendModuleTest.java | 85 +++++++++++ .../s3/actions/S3QueryActionTest.java | 41 +++++ .../filesystem/s3/utils/S3UtilsTest.java | 12 +- 13 files changed, 655 insertions(+), 52 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 78088b8e..e8ae33cf 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -68,7 +69,17 @@ public abstract class AbstractBaseFilesystemAction /******************************************************************************* ** List the files for a table - to be implemented in module-specific subclasses. *******************************************************************************/ - public abstract List listFiles(QTableMetaData table, QBackendMetaData backendBase); + public List listFiles(QTableMetaData table, QBackendMetaData backendBase) throws QException + { + return (listFiles(table, backendBase, null)); + } + + + + /******************************************************************************* + ** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses. + *******************************************************************************/ + public abstract List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException; /******************************************************************************* ** Read the contents of a file - to be implemented in module-specific subclasses. @@ -181,6 +192,7 @@ public abstract class AbstractBaseFilesystemAction /******************************************************************************* ** Generic implementation of the execute method from the QueryInterface *******************************************************************************/ + @SuppressWarnings("checkstyle:Indentation") public QueryOutput executeQuery(QueryInput queryInput) throws QException { preAction(queryInput.getBackend()); @@ -191,51 +203,76 @@ public abstract class AbstractBaseFilesystemAction QTableMetaData table = queryInput.getTable(); AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); - List files = listFiles(table, queryInput.getBackend()); + List files = listFiles(table, queryInput.getBackend(), queryInput.getFilter()); for(FILE file : files) { LOG.info("Processing file: " + getFullPathForFile(file)); - switch(tableDetails.getRecordFormat()) - { - case CSV: - { - String fileContents = IOUtils.toString(readFile(file)); - fileContents = customizeFileContentsAfterReading(table, fileContents); - if(queryInput.getRecordPipe() != null) + InputStream inputStream = readFile(file); + switch(tableDetails.getCardinality()) + { + case MANY: + { + switch(tableDetails.getRecordFormat()) { - new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> + case CSV: { - //////////////////////////////////////////////////////////////////////////////////////////// - // Before the records go into the pipe, make sure their backend details are added to them // - //////////////////////////////////////////////////////////////////////////////////////////// - addBackendDetailsToRecord(record, file); - })); - } - else - { - List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); - addBackendDetailsToRecords(recordsInFile, file); - queryOutput.addRecords(recordsInFile); + String fileContents = IOUtils.toString(inputStream); + fileContents = customizeFileContentsAfterReading(table, fileContents); + + if(queryInput.getRecordPipe() != null) + { + new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> + { + //////////////////////////////////////////////////////////////////////////////////////////// + // Before the records go into the pipe, make sure their backend details are added to them // + //////////////////////////////////////////////////////////////////////////////////////////// + addBackendDetailsToRecord(record, file); + })); + } + else + { + List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + queryOutput.addRecords(recordsInFile); + } + break; + } + case JSON: + { + String fileContents = IOUtils.toString(inputStream); + fileContents = customizeFileContentsAfterReading(table, fileContents); + + // todo - pipe support!! + List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + + queryOutput.addRecords(recordsInFile); + break; + } + default: + { + throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat()); + } } break; } - case JSON: + case ONE: { - String fileContents = IOUtils.toString(readFile(file)); - fileContents = customizeFileContentsAfterReading(table, fileContents); + String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table); - // todo - pipe support!! - List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); - addBackendDetailsToRecords(recordsInFile, file); + byte[] bytes = inputStream.readAllBytes(); + QRecord record = new QRecord() + .withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase) + .withValue(tableDetails.getContentsFieldName(), bytes); + queryOutput.addRecord(record); - queryOutput.addRecords(recordsInFile); break; } default: { - throw new NotImplementedException("Filesystem record format " + tableDetails.getRecordFormat() + " is not yet implemented"); + throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality()); } } } @@ -342,8 +379,8 @@ public abstract class AbstractBaseFilesystemAction { for(QRecord record : insertInput.getRecords()) { - String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString("fileName")); - writeFile(backend, fullPath, record.getValueByteArray("contents")); + String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName())); + writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName())); record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath); output.addRecord(record); } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java index 32bfa2d6..95d4b438 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java @@ -35,6 +35,11 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails private RecordFormat recordFormat; private Cardinality cardinality; + /////////////////////////////////////////////////////////////////////////////////////////////////// + // todo default these to null, and give validation error if not set for a cardinality=ONE table? // + /////////////////////////////////////////////////////////////////////////////////////////////////// + private String contentsFieldName = "contents"; + private String fileNameFieldName = "fileName"; /******************************************************************************* @@ -175,4 +180,67 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails return ((T) this); } + + + /******************************************************************************* + ** Getter for contentsFieldName + *******************************************************************************/ + public String getContentsFieldName() + { + return (this.contentsFieldName); + } + + + + /******************************************************************************* + ** Setter for contentsFieldName + *******************************************************************************/ + public void setContentsFieldName(String contentsFieldName) + { + this.contentsFieldName = contentsFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for contentsFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withContentsFieldName(String contentsFieldName) + { + this.contentsFieldName = contentsFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for fileNameFieldName + *******************************************************************************/ + public String getFileNameFieldName() + { + return (this.fileNameFieldName); + } + + + + /******************************************************************************* + ** Setter for fileNameFieldName + *******************************************************************************/ + public void setFileNameFieldName(String fileNameFieldName) + { + this.fileNameFieldName = fileNameFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for fileNameFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withFileNameFieldName(String fileNameFieldName) + { + this.fileNameFieldName = fileNameFieldName; + return (this); + } + + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index 12a73165..8376d922 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -23,19 +23,36 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions; import java.io.File; +import java.io.FileFilter; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOCase; +import org.apache.commons.io.filefilter.AndFileFilter; +import org.apache.commons.io.filefilter.NameFileFilter; +import org.apache.commons.io.filefilter.OrFileFilter; +import org.apache.commons.io.filefilter.TrueFileFilter; +import org.apache.commons.io.filefilter.WildcardFileFilter; /******************************************************************************* @@ -51,12 +68,66 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction ** List the files for this table. *******************************************************************************/ @Override - public List listFiles(QTableMetaData table, QBackendMetaData backendBase) + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException { - // todo - needs rewritten to do globbing... String fullPath = getFullBasePath(table, backendBase); File directory = new File(fullPath); - File[] files = directory.listFiles(); + File[] files = null; + + AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); + + FileFilter fileFilter = TrueFileFilter.INSTANCE; + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if each file is its own record (ONE), then we may need to do filtering of the directory listing based on the input filter // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(Cardinality.ONE.equals(tableBackendDetails.getCardinality())) + { + if(filter != null && filter.hasAnyCriteria()) + { + List fileFilterList = new ArrayList<>(); + for(QFilterCriteria criteria : filter.getCriteria()) + { + if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName())) + { + if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1) + { + fileFilterList.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(0)))); + } + else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty()) + { + List nameInFilters = new ArrayList<>(); + for(int i = 0; i < criteria.getValues().size(); i++) + { + nameInFilters.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(i)))); + } + fileFilterList.add(new OrFileFilter(nameInFilters)); + } + else + { + throw (new QException("Unable to query filename field using operator: " + criteria.getOperator())); + } + } + else + { + throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName())); + } + } + + fileFilter = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? new AndFileFilter(fileFilterList) : new OrFileFilter(fileFilterList); + } + } + + /////////////////////////////////////////////////////////////////////////////////////////// + // if the table has a glob specified, add it as an AND to the filter built to this point // + /////////////////////////////////////////////////////////////////////////////////////////// + if(StringUtils.hasContent(tableBackendDetails.getGlob())) + { + WildcardFileFilter globFilenameFilter = new WildcardFileFilter(tableBackendDetails.getGlob(), IOCase.INSENSITIVE); + fileFilter = new AndFileFilter(List.of(globFilenameFilter, fileFilter)); + } + + files = directory.listFiles(fileFilter); if(files == null) { diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 612652e7..d059d671 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -30,7 +30,9 @@ import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -126,7 +128,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction listFiles(QTableMetaData table, QBackendMetaData backendBase) + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException { S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase); AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); @@ -138,7 +140,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction listObjectsInBucketMatchingGlob(String bucketName, String path, String glob) + public List listObjectsInBucketMatchingGlob(String bucketName, String path, String glob) throws QException + { + return listObjectsInBucketMatchingGlob(bucketName, path, glob, null, null); + } + + + + /******************************************************************************* + ** List the objects in an S3 bucket matching a glob, per: + ** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String) + ** and also - (possibly) apply a file-name filter (based on the table's details). + *******************************************************************************/ + public List listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException { ////////////////////////////////////////////////////////////////////////////////////////////////// // s3 list requests find nothing if the path starts with a /, so strip away any leading slashes // @@ -77,6 +96,28 @@ public class S3Utils prefix = prefix.substring(0, prefix.indexOf('*')); } + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // for a file-per-record (ONE) table, we may need to apply the filter to listing. // + // but for MANY tables, the filtering would be done on the records after they came out of the files. // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + boolean useQQueryFilter = false; + if(tableDetails != null && Cardinality.ONE.equals(tableDetails.getCardinality())) + { + useQQueryFilter = true; + } + + if(filter != null && useQQueryFilter) + { + if(filter.getCriteria() != null && filter.getCriteria().size() == 1) + { + QFilterCriteria criteria = filter.getCriteria().get(0); + if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS)) + { + prefix += "/" + criteria.getValues().get(0); + } + } + } + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob. // // That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. // @@ -86,6 +127,7 @@ public class S3Utils { glob = ""; } + if(!glob.contains("*")) { if(glob.equals("")) @@ -114,7 +156,7 @@ public class S3Utils { listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken()); } - LOG.info("Listing bucket=" + bucketName + ", path=" + path); + LOG.info("Listing bucket=" + bucketName + ", path=" + path + ", prefix=" + prefix + ", glob=" + glob); listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request); ////////////////////////////////// @@ -149,6 +191,14 @@ public class S3Utils continue; } + /////////////////////////////////////////////////////////////////////////////////// + // if we're a file-per-record table, and we have a filter, compare the key to it // + /////////////////////////////////////////////////////////////////////////////////// + if(!doesObjectKeyMatchFilter(key, filter, tableDetails)) + { + continue; + } + rs.add(objectSummary); } } @@ -159,6 +209,95 @@ public class S3Utils + /******************************************************************************* + ** + *******************************************************************************/ + private boolean doesObjectKeyMatchFilter(String key, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException + { + if(filter == null || !filter.hasAnyCriteria()) + { + return (true); + } + + Path path = Path.of(URI.create("file:///" + key)); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches // + //////////////////////////////////////////////////////////////////////////////////////////////////// + for(QFilterCriteria criteria : filter.getCriteria()) + { + boolean matches = doesObjectKeyMatchOneCriteria(criteria, tableDetails, path); + + if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator())) + { + //////////////////////////////////////////////////////////////////////////////// + // if it's not a match, and it's an AND filter, then the whole thing is false // + //////////////////////////////////////////////////////////////////////////////// + return (false); + } + + if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator())) + { + //////////////////////////////////////////////////////////// + // if it's an OR filter, and we've a match, return a true // + //////////////////////////////////////////////////////////// + return (true); + } + } + + ////////////////////////////////////////////////////////////////////// + // if we didn't return above, return now // + // for an OR - if we didn't find something true, then return false. // + // else, an AND - if we didn't find a false, we can return true. // + ////////////////////////////////////////////////////////////////////// + if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator())) + { + return (false); + } + + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static boolean doesObjectKeyMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException + { + if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName())) + { + if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1) + { + return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path)); + } + else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty()) + { + boolean anyMatch = false; + for(int i = 0; i < criteria.getValues().size(); i++) + { + if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path)) + { + anyMatch = true; + break; + } + } + + return (anyMatch); + } + else + { + throw (new QException("Unable to query filename field using operator: " + criteria.getOperator())); + } + } + else + { + throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName())); + } + } + + + /******************************************************************************* ** Get the contents (as an InputStream) for an object in s3 *******************************************************************************/ @@ -245,4 +384,5 @@ public class S3Utils return amazonS3; } + } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java index ea52bc97..f659b265 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java @@ -25,6 +25,11 @@ package com.kingsrook.qqq.backend.module.filesystem.local; import java.io.File; import java.io.IOException; import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; @@ -36,6 +41,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -47,6 +54,9 @@ public class FilesystemBackendModuleTest + /******************************************************************************* + ** + *******************************************************************************/ @BeforeEach public void beforeEach() throws IOException { @@ -55,6 +65,9 @@ public class FilesystemBackendModuleTest + /******************************************************************************* + ** + *******************************************************************************/ @AfterEach public void afterEach() throws Exception { @@ -63,6 +76,78 @@ public class FilesystemBackendModuleTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testListFiles() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS); + + AbstractFilesystemAction abstractFilesystemAction = new AbstractFilesystemAction(); + QBackendMetaData backend = qInstance.getBackendForTable(table.getName()); + + ////////////////////////////////////////////////////////// + // with no filter given, all (3) files should come back // + ////////////////////////////////////////////////////////// + List files = abstractFilesystemAction.listFiles(table, backend); + assertEquals(3, files.size()); + + ///////////////////////////////////////// + // filter for a file name that's found // + ///////////////////////////////////////// + files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + assertEquals(1, files.size()); + assertEquals("BLOB-2.txt", files.get(0).getName()); + + files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); + assertEquals(1, files.size()); + assertEquals("BLOB-1.txt", files.get(0).getName()); + + /////////////////////////////////// + // filter for 2 names that exist // + /////////////////////////////////// + files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); + assertEquals(2, files.size()); + + ///////////////////////////////////////////// + // filter for a file name that isn't found // + ///////////////////////////////////////////// + files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt"))); + assertEquals(0, files.size()); + + files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); + assertEquals(1, files.size()); + + //////////////////////////////////////////////////// + // 2 criteria, and'ed, and can't match, so find 0 // + //////////////////////////////////////////////////// + files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + assertEquals(0, files.size()); + + ////////////////////////////////////////////////// + // 2 criteria, or'ed, and both match, so find 2 // + ////////////////////////////////////////////////// + files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) + .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); + assertEquals(2, files.size()); + + ////////////////////////////////////// + // ensure unsupported filters throw // + ////////////////////////////////////// + assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42)))) + .hasMessageContaining("Unable to query filesystem table by field"); + assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK)))) + .hasMessageContaining("Unable to query filename field using operator"); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java index 1a1fac9e..b4117b4e 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java @@ -89,6 +89,7 @@ public class FilesystemActionTest extends BaseTest writePersonJSONFiles(baseDirectory); writePersonCSVFiles(baseDirectory); + writeBlobFiles(baseDirectory); } @@ -158,6 +159,37 @@ public class FilesystemActionTest extends BaseTest + /******************************************************************************* + ** Write some data files into the directory for the filesystem module. + *******************************************************************************/ + private void writeBlobFiles(File baseDirectory) throws IOException + { + String fullPath = baseDirectory.getAbsolutePath(); + if(TestUtils.defineLocalFilesystemBlobTable().getBackendDetails() instanceof FilesystemTableBackendDetails details) + { + if(StringUtils.hasContent(details.getBasePath())) + { + fullPath += File.separatorChar + details.getBasePath(); + } + } + fullPath += File.separatorChar; + + String data1 = """ + Hello, Blob + """; + FileUtils.writeStringToFile(new File(fullPath + "BLOB-1.txt"), data1); + + String data2 = """ + Hi Bob"""; + FileUtils.writeStringToFile(new File(fullPath + "BLOB-2.txt"), data2); + + String data3 = """ + # Hi MD..."""; + FileUtils.writeStringToFile(new File(fullPath + "BLOB-3.md"), data3); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java index fdf79b2b..26d6629f 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java @@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -32,8 +35,10 @@ import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractPostReadFileCustomizer; import com.kingsrook.qqq.backend.module.filesystem.base.actions.FilesystemTableCustomizers; -import org.junit.jupiter.api.Assertions; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* @@ -51,8 +56,8 @@ public class FilesystemQueryActionTest extends FilesystemActionTest QueryInput queryInput = new QueryInput(); queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName()); QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput); - Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); - Assertions.assertTrue(queryOutput.getRecords().stream() + assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); + assertTrue(queryOutput.getRecords().stream() .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)), "All records should have a full-path in their backend details, matching the test folder name"); } @@ -74,14 +79,48 @@ public class FilesystemQueryActionTest extends FilesystemActionTest queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName()); QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput); - Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); - Assertions.assertTrue( + assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); + assertTrue( queryOutput.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")), "All records should have their email addresses up-shifted."); } + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQueryForCardinalityOne() throws QException + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS); + queryInput.setFilter(new QQueryFilter()); + QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); + queryOutput = new FilesystemQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row"); + assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName")); + + //////////////////////////////////////////////////////////////// + // put a glob on the table - now should only find 2 txt files // + //////////////////////////////////////////////////////////////// + QInstance instance = TestUtils.defineInstance(); + ((FilesystemTableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).getBackendDetails())) + .withGlob("*.txt"); + reInitInstanceInContext(instance); + + queryInput.setFilter(new QQueryFilter()); + queryOutput = new FilesystemQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ public static class ValueUpshifter extends AbstractPostReadFileCustomizer { @Override diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java index a4c9e7f0..1ef2f8d9 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java @@ -26,7 +26,7 @@ import java.util.List; import java.util.stream.Collectors; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction; -import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -187,7 +187,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test /******************************************************************************* ** *******************************************************************************/ - private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException + private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QException { S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend); AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase(); @@ -207,7 +207,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test /******************************************************************************* ** *******************************************************************************/ - private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException + private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QException { S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend); AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase(); diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java index a7585d86..e9538d4b 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java @@ -62,6 +62,9 @@ public class BaseS3Test extends BaseTest amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2()); amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test"); amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3()); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-1.txt", "Hello, Blob"); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-2.txt", "Hi, Bob"); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-3.md", "# Hi, MD"); } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java index 8474e5ae..644c4575 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java @@ -25,6 +25,11 @@ package com.kingsrook.qqq.backend.module.filesystem.s3; import java.util.List; import java.util.UUID; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; @@ -32,6 +37,9 @@ import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemExceptio import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -43,6 +51,83 @@ public class S3BackendModuleTest extends BaseS3Test + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testListFiles() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_S3); + QBackendMetaData backend = qInstance.getBackendForTable(table.getName()); + + ////////////////////////////////////////////////////// + // set up the backend module (e.g., for localstack) // + ////////////////////////////////////////////////////// + S3BackendModule s3BackendModule = new S3BackendModule(); + AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); + actionBase.setS3Utils(getS3Utils()); + + ////////////////////////////////////////////////////////// + // with no filter given, all (3) files should come back // + ////////////////////////////////////////////////////////// + List files = actionBase.listFiles(table, backend); + assertEquals(3, files.size()); + + ///////////////////////////////////////// + // filter for a file name that's found // + ///////////////////////////////////////// + files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + assertEquals(1, files.size()); + assertThat(files.get(0).getKey()).contains("BLOB-2.txt"); + + files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); + assertEquals(1, files.size()); + assertThat(files.get(0).getKey()).contains("BLOB-1.txt"); + + /////////////////////////////////// + // filter for 2 names that exist // + /////////////////////////////////// + files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); + assertEquals(2, files.size()); + + ///////////////////////////////////////////// + // filter for a file name that isn't found // + ///////////////////////////////////////////// + files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt"))); + assertEquals(0, files.size()); + + files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); + assertEquals(1, files.size()); + + //////////////////////////////////////////////////// + // 2 criteria, and'ed, and can't match, so find 0 // + //////////////////////////////////////////////////// + files = actionBase.listFiles(table, backend, new QQueryFilter( + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + assertEquals(0, files.size()); + + ////////////////////////////////////////////////// + // 2 criteria, or'ed, and both match, so find 2 // + ////////////////////////////////////////////////// + files = actionBase.listFiles(table, backend, new QQueryFilter( + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) + .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); + assertEquals(2, files.size()); + + ////////////////////////////////////// + // ensure unsupported filters throw // + ////////////////////////////////////// + assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42)))) + .hasMessageContaining("Unable to query filesystem table by field"); + assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK)))) + .hasMessageContaining("Unable to query filename field using operator"); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java index 9149686e..36a7c771 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java @@ -23,13 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -66,4 +72,39 @@ public class S3QueryActionTest extends BaseS3Test return queryInput; } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQueryForCardinalityOne() throws QException + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_S3); + queryInput.setFilter(new QQueryFilter()); + + S3QueryAction s3QueryAction = new S3QueryAction(); + s3QueryAction.setS3Utils(getS3Utils()); + + QueryOutput queryOutput = s3QueryAction.execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); + queryOutput = s3QueryAction.execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row"); + assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName")); + + //////////////////////////////////////////////////////////////// + // put a glob on the table - now should only find 2 txt files // + //////////////////////////////////////////////////////////////// + QInstance instance = TestUtils.defineInstance(); + ((S3TableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_S3).getBackendDetails())) + .withGlob("*.txt"); + reInitInstanceInContext(instance); + + queryInput.setFilter(new QQueryFilter()); + queryOutput = s3QueryAction.execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows"); + } + } \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java index ed4afbdf..cc1c007f 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java @@ -22,10 +22,10 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.utils; -import java.io.IOException; import java.io.InputStream; import java.util.List; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.Test; @@ -42,14 +42,14 @@ public class S3UtilsTest extends BaseS3Test ** *******************************************************************************/ @Test - public void testListObjectsInBucketAtPath() + public void testListObjectsInBucketAtPath() throws QException { S3Utils s3Utils = getS3Utils(); assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders"); assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders"); assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders"); assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders"); - assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders"); + assertEquals(7, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders"); assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash"); assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash"); assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes"); @@ -60,8 +60,8 @@ public class S3UtilsTest extends BaseS3Test assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /"); assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s"); assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string"); - assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively"); - assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively"); + assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively"); + assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively"); } @@ -70,7 +70,7 @@ public class S3UtilsTest extends BaseS3Test ** *******************************************************************************/ @Test - public void testGetObjectAsInputStream() throws IOException + public void testGetObjectAsInputStream() throws Exception { S3Utils s3Utils = getS3Utils(); List s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", "");