From 940080bc865c33b1de4196bb5d7a060e8207b52f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Dec 2023 16:10:45 -0600 Subject: [PATCH 1/6] CE-773 update to support listing/filtering filesystem tables with Cardinality.ONE (single-record per-file) --- .../actions/AbstractBaseFilesystemAction.java | 99 ++++++++---- ...AbstractFilesystemTableBackendDetails.java | 68 +++++++++ .../actions/AbstractFilesystemAction.java | 77 +++++++++- .../s3/actions/AbstractS3Action.java | 6 +- .../module/filesystem/s3/utils/S3Utils.java | 144 +++++++++++++++++- .../local/FilesystemBackendModuleTest.java | 85 +++++++++++ .../local/actions/FilesystemActionTest.java | 32 ++++ .../actions/FilesystemQueryActionTest.java | 49 +++++- .../sync/FilesystemSyncProcessS3Test.java | 6 +- .../module/filesystem/s3/BaseS3Test.java | 3 + .../filesystem/s3/S3BackendModuleTest.java | 85 +++++++++++ .../s3/actions/S3QueryActionTest.java | 41 +++++ .../filesystem/s3/utils/S3UtilsTest.java | 12 +- 13 files changed, 655 insertions(+), 52 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 78088b8e..e8ae33cf 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -68,7 +69,17 @@ public abstract class AbstractBaseFilesystemAction /******************************************************************************* ** List the files for a table - to be implemented in module-specific subclasses. *******************************************************************************/ - public abstract List listFiles(QTableMetaData table, QBackendMetaData backendBase); + public List listFiles(QTableMetaData table, QBackendMetaData backendBase) throws QException + { + return (listFiles(table, backendBase, null)); + } + + + + /******************************************************************************* + ** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses. + *******************************************************************************/ + public abstract List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException; /******************************************************************************* ** Read the contents of a file - to be implemented in module-specific subclasses. @@ -181,6 +192,7 @@ public abstract class AbstractBaseFilesystemAction /******************************************************************************* ** Generic implementation of the execute method from the QueryInterface *******************************************************************************/ + @SuppressWarnings("checkstyle:Indentation") public QueryOutput executeQuery(QueryInput queryInput) throws QException { preAction(queryInput.getBackend()); @@ -191,51 +203,76 @@ public abstract class AbstractBaseFilesystemAction QTableMetaData table = queryInput.getTable(); AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); - List files = listFiles(table, queryInput.getBackend()); + List files = listFiles(table, queryInput.getBackend(), queryInput.getFilter()); for(FILE file : files) { LOG.info("Processing file: " + getFullPathForFile(file)); - switch(tableDetails.getRecordFormat()) - { - case CSV: - { - String fileContents = IOUtils.toString(readFile(file)); - fileContents = customizeFileContentsAfterReading(table, fileContents); - if(queryInput.getRecordPipe() != null) + InputStream inputStream = readFile(file); + switch(tableDetails.getCardinality()) + { + case MANY: + { + switch(tableDetails.getRecordFormat()) { - new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> + case CSV: { - //////////////////////////////////////////////////////////////////////////////////////////// - // Before the records go into the pipe, make sure their backend details are added to them // - //////////////////////////////////////////////////////////////////////////////////////////// - addBackendDetailsToRecord(record, file); - })); - } - else - { - List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); - addBackendDetailsToRecords(recordsInFile, file); - queryOutput.addRecords(recordsInFile); + String fileContents = IOUtils.toString(inputStream); + fileContents = customizeFileContentsAfterReading(table, fileContents); + + if(queryInput.getRecordPipe() != null) + { + new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> + { + //////////////////////////////////////////////////////////////////////////////////////////// + // Before the records go into the pipe, make sure their backend details are added to them // + //////////////////////////////////////////////////////////////////////////////////////////// + addBackendDetailsToRecord(record, file); + })); + } + else + { + List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + queryOutput.addRecords(recordsInFile); + } + break; + } + case JSON: + { + String fileContents = IOUtils.toString(inputStream); + fileContents = customizeFileContentsAfterReading(table, fileContents); + + // todo - pipe support!! + List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + + queryOutput.addRecords(recordsInFile); + break; + } + default: + { + throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat()); + } } break; } - case JSON: + case ONE: { - String fileContents = IOUtils.toString(readFile(file)); - fileContents = customizeFileContentsAfterReading(table, fileContents); + String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table); - // todo - pipe support!! - List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); - addBackendDetailsToRecords(recordsInFile, file); + byte[] bytes = inputStream.readAllBytes(); + QRecord record = new QRecord() + .withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase) + .withValue(tableDetails.getContentsFieldName(), bytes); + queryOutput.addRecord(record); - queryOutput.addRecords(recordsInFile); break; } default: { - throw new NotImplementedException("Filesystem record format " + tableDetails.getRecordFormat() + " is not yet implemented"); + throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality()); } } } @@ -342,8 +379,8 @@ public abstract class AbstractBaseFilesystemAction { for(QRecord record : insertInput.getRecords()) { - String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString("fileName")); - writeFile(backend, fullPath, record.getValueByteArray("contents")); + String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName())); + writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName())); record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath); output.addRecord(record); } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java index 32bfa2d6..95d4b438 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java @@ -35,6 +35,11 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails private RecordFormat recordFormat; private Cardinality cardinality; + /////////////////////////////////////////////////////////////////////////////////////////////////// + // todo default these to null, and give validation error if not set for a cardinality=ONE table? // + /////////////////////////////////////////////////////////////////////////////////////////////////// + private String contentsFieldName = "contents"; + private String fileNameFieldName = "fileName"; /******************************************************************************* @@ -175,4 +180,67 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails return ((T) this); } + + + /******************************************************************************* + ** Getter for contentsFieldName + *******************************************************************************/ + public String getContentsFieldName() + { + return (this.contentsFieldName); + } + + + + /******************************************************************************* + ** Setter for contentsFieldName + *******************************************************************************/ + public void setContentsFieldName(String contentsFieldName) + { + this.contentsFieldName = contentsFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for contentsFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withContentsFieldName(String contentsFieldName) + { + this.contentsFieldName = contentsFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for fileNameFieldName + *******************************************************************************/ + public String getFileNameFieldName() + { + return (this.fileNameFieldName); + } + + + + /******************************************************************************* + ** Setter for fileNameFieldName + *******************************************************************************/ + public void setFileNameFieldName(String fileNameFieldName) + { + this.fileNameFieldName = fileNameFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for fileNameFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withFileNameFieldName(String fileNameFieldName) + { + this.fileNameFieldName = fileNameFieldName; + return (this); + } + + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index 12a73165..8376d922 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -23,19 +23,36 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions; import java.io.File; +import java.io.FileFilter; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOCase; +import org.apache.commons.io.filefilter.AndFileFilter; +import org.apache.commons.io.filefilter.NameFileFilter; +import org.apache.commons.io.filefilter.OrFileFilter; +import org.apache.commons.io.filefilter.TrueFileFilter; +import org.apache.commons.io.filefilter.WildcardFileFilter; /******************************************************************************* @@ -51,12 +68,66 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction ** List the files for this table. *******************************************************************************/ @Override - public List listFiles(QTableMetaData table, QBackendMetaData backendBase) + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException { - // todo - needs rewritten to do globbing... String fullPath = getFullBasePath(table, backendBase); File directory = new File(fullPath); - File[] files = directory.listFiles(); + File[] files = null; + + AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); + + FileFilter fileFilter = TrueFileFilter.INSTANCE; + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if each file is its own record (ONE), then we may need to do filtering of the directory listing based on the input filter // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(Cardinality.ONE.equals(tableBackendDetails.getCardinality())) + { + if(filter != null && filter.hasAnyCriteria()) + { + List fileFilterList = new ArrayList<>(); + for(QFilterCriteria criteria : filter.getCriteria()) + { + if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName())) + { + if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1) + { + fileFilterList.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(0)))); + } + else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty()) + { + List nameInFilters = new ArrayList<>(); + for(int i = 0; i < criteria.getValues().size(); i++) + { + nameInFilters.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(i)))); + } + fileFilterList.add(new OrFileFilter(nameInFilters)); + } + else + { + throw (new QException("Unable to query filename field using operator: " + criteria.getOperator())); + } + } + else + { + throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName())); + } + } + + fileFilter = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? new AndFileFilter(fileFilterList) : new OrFileFilter(fileFilterList); + } + } + + /////////////////////////////////////////////////////////////////////////////////////////// + // if the table has a glob specified, add it as an AND to the filter built to this point // + /////////////////////////////////////////////////////////////////////////////////////////// + if(StringUtils.hasContent(tableBackendDetails.getGlob())) + { + WildcardFileFilter globFilenameFilter = new WildcardFileFilter(tableBackendDetails.getGlob(), IOCase.INSENSITIVE); + fileFilter = new AndFileFilter(List.of(globFilenameFilter, fileFilter)); + } + + files = directory.listFiles(fileFilter); if(files == null) { diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 612652e7..d059d671 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -30,7 +30,9 @@ import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -126,7 +128,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction listFiles(QTableMetaData table, QBackendMetaData backendBase) + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException { S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase); AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); @@ -138,7 +140,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction listObjectsInBucketMatchingGlob(String bucketName, String path, String glob) + public List listObjectsInBucketMatchingGlob(String bucketName, String path, String glob) throws QException + { + return listObjectsInBucketMatchingGlob(bucketName, path, glob, null, null); + } + + + + /******************************************************************************* + ** List the objects in an S3 bucket matching a glob, per: + ** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String) + ** and also - (possibly) apply a file-name filter (based on the table's details). + *******************************************************************************/ + public List listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException { ////////////////////////////////////////////////////////////////////////////////////////////////// // s3 list requests find nothing if the path starts with a /, so strip away any leading slashes // @@ -77,6 +96,28 @@ public class S3Utils prefix = prefix.substring(0, prefix.indexOf('*')); } + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // for a file-per-record (ONE) table, we may need to apply the filter to listing. // + // but for MANY tables, the filtering would be done on the records after they came out of the files. // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + boolean useQQueryFilter = false; + if(tableDetails != null && Cardinality.ONE.equals(tableDetails.getCardinality())) + { + useQQueryFilter = true; + } + + if(filter != null && useQQueryFilter) + { + if(filter.getCriteria() != null && filter.getCriteria().size() == 1) + { + QFilterCriteria criteria = filter.getCriteria().get(0); + if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS)) + { + prefix += "/" + criteria.getValues().get(0); + } + } + } + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob. // // That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. // @@ -86,6 +127,7 @@ public class S3Utils { glob = ""; } + if(!glob.contains("*")) { if(glob.equals("")) @@ -114,7 +156,7 @@ public class S3Utils { listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken()); } - LOG.info("Listing bucket=" + bucketName + ", path=" + path); + LOG.info("Listing bucket=" + bucketName + ", path=" + path + ", prefix=" + prefix + ", glob=" + glob); listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request); ////////////////////////////////// @@ -149,6 +191,14 @@ public class S3Utils continue; } + /////////////////////////////////////////////////////////////////////////////////// + // if we're a file-per-record table, and we have a filter, compare the key to it // + /////////////////////////////////////////////////////////////////////////////////// + if(!doesObjectKeyMatchFilter(key, filter, tableDetails)) + { + continue; + } + rs.add(objectSummary); } } @@ -159,6 +209,95 @@ public class S3Utils + /******************************************************************************* + ** + *******************************************************************************/ + private boolean doesObjectKeyMatchFilter(String key, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException + { + if(filter == null || !filter.hasAnyCriteria()) + { + return (true); + } + + Path path = Path.of(URI.create("file:///" + key)); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches // + //////////////////////////////////////////////////////////////////////////////////////////////////// + for(QFilterCriteria criteria : filter.getCriteria()) + { + boolean matches = doesObjectKeyMatchOneCriteria(criteria, tableDetails, path); + + if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator())) + { + //////////////////////////////////////////////////////////////////////////////// + // if it's not a match, and it's an AND filter, then the whole thing is false // + //////////////////////////////////////////////////////////////////////////////// + return (false); + } + + if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator())) + { + //////////////////////////////////////////////////////////// + // if it's an OR filter, and we've a match, return a true // + //////////////////////////////////////////////////////////// + return (true); + } + } + + ////////////////////////////////////////////////////////////////////// + // if we didn't return above, return now // + // for an OR - if we didn't find something true, then return false. // + // else, an AND - if we didn't find a false, we can return true. // + ////////////////////////////////////////////////////////////////////// + if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator())) + { + return (false); + } + + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static boolean doesObjectKeyMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException + { + if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName())) + { + if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1) + { + return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path)); + } + else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty()) + { + boolean anyMatch = false; + for(int i = 0; i < criteria.getValues().size(); i++) + { + if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path)) + { + anyMatch = true; + break; + } + } + + return (anyMatch); + } + else + { + throw (new QException("Unable to query filename field using operator: " + criteria.getOperator())); + } + } + else + { + throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName())); + } + } + + + /******************************************************************************* ** Get the contents (as an InputStream) for an object in s3 *******************************************************************************/ @@ -245,4 +384,5 @@ public class S3Utils return amazonS3; } + } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java index ea52bc97..f659b265 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java @@ -25,6 +25,11 @@ package com.kingsrook.qqq.backend.module.filesystem.local; import java.io.File; import java.io.IOException; import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; @@ -36,6 +41,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -47,6 +54,9 @@ public class FilesystemBackendModuleTest + /******************************************************************************* + ** + *******************************************************************************/ @BeforeEach public void beforeEach() throws IOException { @@ -55,6 +65,9 @@ public class FilesystemBackendModuleTest + /******************************************************************************* + ** + *******************************************************************************/ @AfterEach public void afterEach() throws Exception { @@ -63,6 +76,78 @@ public class FilesystemBackendModuleTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testListFiles() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS); + + AbstractFilesystemAction abstractFilesystemAction = new AbstractFilesystemAction(); + QBackendMetaData backend = qInstance.getBackendForTable(table.getName()); + + ////////////////////////////////////////////////////////// + // with no filter given, all (3) files should come back // + ////////////////////////////////////////////////////////// + List files = abstractFilesystemAction.listFiles(table, backend); + assertEquals(3, files.size()); + + ///////////////////////////////////////// + // filter for a file name that's found // + ///////////////////////////////////////// + files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + assertEquals(1, files.size()); + assertEquals("BLOB-2.txt", files.get(0).getName()); + + files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); + assertEquals(1, files.size()); + assertEquals("BLOB-1.txt", files.get(0).getName()); + + /////////////////////////////////// + // filter for 2 names that exist // + /////////////////////////////////// + files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); + assertEquals(2, files.size()); + + ///////////////////////////////////////////// + // filter for a file name that isn't found // + ///////////////////////////////////////////// + files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt"))); + assertEquals(0, files.size()); + + files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); + assertEquals(1, files.size()); + + //////////////////////////////////////////////////// + // 2 criteria, and'ed, and can't match, so find 0 // + //////////////////////////////////////////////////// + files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + assertEquals(0, files.size()); + + ////////////////////////////////////////////////// + // 2 criteria, or'ed, and both match, so find 2 // + ////////////////////////////////////////////////// + files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) + .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); + assertEquals(2, files.size()); + + ////////////////////////////////////// + // ensure unsupported filters throw // + ////////////////////////////////////// + assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42)))) + .hasMessageContaining("Unable to query filesystem table by field"); + assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK)))) + .hasMessageContaining("Unable to query filename field using operator"); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java index 1a1fac9e..b4117b4e 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java @@ -89,6 +89,7 @@ public class FilesystemActionTest extends BaseTest writePersonJSONFiles(baseDirectory); writePersonCSVFiles(baseDirectory); + writeBlobFiles(baseDirectory); } @@ -158,6 +159,37 @@ public class FilesystemActionTest extends BaseTest + /******************************************************************************* + ** Write some data files into the directory for the filesystem module. + *******************************************************************************/ + private void writeBlobFiles(File baseDirectory) throws IOException + { + String fullPath = baseDirectory.getAbsolutePath(); + if(TestUtils.defineLocalFilesystemBlobTable().getBackendDetails() instanceof FilesystemTableBackendDetails details) + { + if(StringUtils.hasContent(details.getBasePath())) + { + fullPath += File.separatorChar + details.getBasePath(); + } + } + fullPath += File.separatorChar; + + String data1 = """ + Hello, Blob + """; + FileUtils.writeStringToFile(new File(fullPath + "BLOB-1.txt"), data1); + + String data2 = """ + Hi Bob"""; + FileUtils.writeStringToFile(new File(fullPath + "BLOB-2.txt"), data2); + + String data3 = """ + # Hi MD..."""; + FileUtils.writeStringToFile(new File(fullPath + "BLOB-3.md"), data3); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java index fdf79b2b..26d6629f 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java @@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -32,8 +35,10 @@ import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractPostReadFileCustomizer; import com.kingsrook.qqq.backend.module.filesystem.base.actions.FilesystemTableCustomizers; -import org.junit.jupiter.api.Assertions; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* @@ -51,8 +56,8 @@ public class FilesystemQueryActionTest extends FilesystemActionTest QueryInput queryInput = new QueryInput(); queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName()); QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput); - Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); - Assertions.assertTrue(queryOutput.getRecords().stream() + assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); + assertTrue(queryOutput.getRecords().stream() .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)), "All records should have a full-path in their backend details, matching the test folder name"); } @@ -74,14 +79,48 @@ public class FilesystemQueryActionTest extends FilesystemActionTest queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName()); QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput); - Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); - Assertions.assertTrue( + assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); + assertTrue( queryOutput.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")), "All records should have their email addresses up-shifted."); } + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQueryForCardinalityOne() throws QException + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS); + queryInput.setFilter(new QQueryFilter()); + QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); + queryOutput = new FilesystemQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row"); + assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName")); + + //////////////////////////////////////////////////////////////// + // put a glob on the table - now should only find 2 txt files // + //////////////////////////////////////////////////////////////// + QInstance instance = TestUtils.defineInstance(); + ((FilesystemTableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).getBackendDetails())) + .withGlob("*.txt"); + reInitInstanceInContext(instance); + + queryInput.setFilter(new QQueryFilter()); + queryOutput = new FilesystemQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ public static class ValueUpshifter extends AbstractPostReadFileCustomizer { @Override diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java index a4c9e7f0..1ef2f8d9 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java @@ -26,7 +26,7 @@ import java.util.List; import java.util.stream.Collectors; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction; -import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -187,7 +187,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test /******************************************************************************* ** *******************************************************************************/ - private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException + private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QException { S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend); AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase(); @@ -207,7 +207,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test /******************************************************************************* ** *******************************************************************************/ - private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException + private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QException { S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend); AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase(); diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java index a7585d86..e9538d4b 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java @@ -62,6 +62,9 @@ public class BaseS3Test extends BaseTest amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2()); amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test"); amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3()); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-1.txt", "Hello, Blob"); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-2.txt", "Hi, Bob"); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-3.md", "# Hi, MD"); } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java index 8474e5ae..644c4575 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java @@ -25,6 +25,11 @@ package com.kingsrook.qqq.backend.module.filesystem.s3; import java.util.List; import java.util.UUID; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; @@ -32,6 +37,9 @@ import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemExceptio import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -43,6 +51,83 @@ public class S3BackendModuleTest extends BaseS3Test + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testListFiles() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_S3); + QBackendMetaData backend = qInstance.getBackendForTable(table.getName()); + + ////////////////////////////////////////////////////// + // set up the backend module (e.g., for localstack) // + ////////////////////////////////////////////////////// + S3BackendModule s3BackendModule = new S3BackendModule(); + AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); + actionBase.setS3Utils(getS3Utils()); + + ////////////////////////////////////////////////////////// + // with no filter given, all (3) files should come back // + ////////////////////////////////////////////////////////// + List files = actionBase.listFiles(table, backend); + assertEquals(3, files.size()); + + ///////////////////////////////////////// + // filter for a file name that's found // + ///////////////////////////////////////// + files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + assertEquals(1, files.size()); + assertThat(files.get(0).getKey()).contains("BLOB-2.txt"); + + files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); + assertEquals(1, files.size()); + assertThat(files.get(0).getKey()).contains("BLOB-1.txt"); + + /////////////////////////////////// + // filter for 2 names that exist // + /////////////////////////////////// + files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); + assertEquals(2, files.size()); + + ///////////////////////////////////////////// + // filter for a file name that isn't found // + ///////////////////////////////////////////// + files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt"))); + assertEquals(0, files.size()); + + files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); + assertEquals(1, files.size()); + + //////////////////////////////////////////////////// + // 2 criteria, and'ed, and can't match, so find 0 // + //////////////////////////////////////////////////// + files = actionBase.listFiles(table, backend, new QQueryFilter( + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + assertEquals(0, files.size()); + + ////////////////////////////////////////////////// + // 2 criteria, or'ed, and both match, so find 2 // + ////////////////////////////////////////////////// + files = actionBase.listFiles(table, backend, new QQueryFilter( + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) + .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); + assertEquals(2, files.size()); + + ////////////////////////////////////// + // ensure unsupported filters throw // + ////////////////////////////////////// + assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42)))) + .hasMessageContaining("Unable to query filesystem table by field"); + assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK)))) + .hasMessageContaining("Unable to query filename field using operator"); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java index 9149686e..36a7c771 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java @@ -23,13 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -66,4 +72,39 @@ public class S3QueryActionTest extends BaseS3Test return queryInput; } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQueryForCardinalityOne() throws QException + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_S3); + queryInput.setFilter(new QQueryFilter()); + + S3QueryAction s3QueryAction = new S3QueryAction(); + s3QueryAction.setS3Utils(getS3Utils()); + + QueryOutput queryOutput = s3QueryAction.execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); + queryOutput = s3QueryAction.execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row"); + assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName")); + + //////////////////////////////////////////////////////////////// + // put a glob on the table - now should only find 2 txt files // + //////////////////////////////////////////////////////////////// + QInstance instance = TestUtils.defineInstance(); + ((S3TableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_S3).getBackendDetails())) + .withGlob("*.txt"); + reInitInstanceInContext(instance); + + queryInput.setFilter(new QQueryFilter()); + queryOutput = s3QueryAction.execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows"); + } + } \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java index ed4afbdf..cc1c007f 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java @@ -22,10 +22,10 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.utils; -import java.io.IOException; import java.io.InputStream; import java.util.List; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.Test; @@ -42,14 +42,14 @@ public class S3UtilsTest extends BaseS3Test ** *******************************************************************************/ @Test - public void testListObjectsInBucketAtPath() + public void testListObjectsInBucketAtPath() throws QException { S3Utils s3Utils = getS3Utils(); assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders"); assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders"); assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders"); assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders"); - assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders"); + assertEquals(7, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders"); assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash"); assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash"); assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes"); @@ -60,8 +60,8 @@ public class S3UtilsTest extends BaseS3Test assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /"); assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s"); assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string"); - assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively"); - assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively"); + assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively"); + assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively"); } @@ -70,7 +70,7 @@ public class S3UtilsTest extends BaseS3Test ** *******************************************************************************/ @Test - public void testGetObjectAsInputStream() throws IOException + public void testGetObjectAsInputStream() throws Exception { S3Utils s3Utils = getS3Utils(); List s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", ""); From b805e7645bdc38b073747ce58f6984ae5116191c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Dec 2023 16:11:19 -0600 Subject: [PATCH 2/6] CE-773 Update for compat. with previous commit, but also, fix all generics and move inputStream into try-with-resources --- .../filesystem/sync/FilesystemSyncStep.java | 72 +++++++++++-------- 1 file changed, 44 insertions(+), 28 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java index 25806028..2796a37e 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java @@ -59,31 +59,45 @@ public class FilesystemSyncStep implements BackendStep *******************************************************************************/ @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // defer to a private method here, so we can add a type-parameter for that method to use // + // would think we could do that here, but get compiler error, since this method comes from base class // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + doRun(runBackendStepInput, runBackendStepOutput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void doRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { QTableMetaData sourceTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE)); QTableMetaData archiveTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE)); QTableMetaData processingTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE)); - QBackendMetaData sourceBackend = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName()); - FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend); - AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase(); + QBackendMetaData sourceBackend = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName()); + FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend); + AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase(); sourceActionBase.preAction(sourceBackend); - Map sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend); + Map sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend); - QBackendMetaData archiveBackend = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName()); - FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend); - AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase(); + QBackendMetaData archiveBackend = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName()); + FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend); + AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase(); archiveActionBase.preAction(archiveBackend); Set archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet(); - QBackendMetaData processingBackend = runBackendStepInput.getInstance().getBackendForTable(processingTable.getName()); - FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend); - AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase(); + QBackendMetaData processingBackend = runBackendStepInput.getInstance().getBackendForTable(processingTable.getName()); + FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend); + AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase(); processingActionBase.preAction(processingBackend); Integer maxFilesToSync = runBackendStepInput.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE); int syncedFileCount = 0; - for(Map.Entry sourceEntry : sourceFiles.entrySet()) + for(Map.Entry sourceEntry : sourceFiles.entrySet()) { try { @@ -91,20 +105,22 @@ public class FilesystemSyncStep implements BackendStep if(!archiveFiles.contains(sourceFileName)) { LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]"); - InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue()); - byte[] bytes = inputStream.readAllBytes(); - - String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend); - archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes); - - String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend); - processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes); - syncedFileCount++; - - if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync) + try(InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue())) { - LOG.info("Breaking after syncing " + syncedFileCount + " files"); - break; + byte[] bytes = inputStream.readAllBytes(); + + String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend); + archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes); + + String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend); + processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes); + syncedFileCount++; + + if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync) + { + LOG.info("Breaking after syncing " + syncedFileCount + " files"); + break; + } } } } @@ -120,12 +136,12 @@ public class FilesystemSyncStep implements BackendStep /******************************************************************************* ** *******************************************************************************/ - private Map getFileNames(AbstractBaseFilesystemAction actionBase, QTableMetaData table, QBackendMetaData backend) + private Map getFileNames(AbstractBaseFilesystemAction actionBase, QTableMetaData table, QBackendMetaData backend) throws QException { - List files = actionBase.listFiles(table, backend); - Map rs = new LinkedHashMap<>(); + List files = actionBase.listFiles(table, backend); + Map rs = new LinkedHashMap<>(); - for(Object file : files) + for(F file : files) { String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table); rs.put(fileName, file); From 345d8022c16d59c4c471263b0eeae0e9eedb6d51 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 28 Dec 2023 10:32:26 -0600 Subject: [PATCH 3/6] CE-773 Feedback from code review --- .../actions/AbstractBaseFilesystemAction.java | 31 +++++++++++++++++-- .../actions/AbstractFilesystemAction.java | 8 +++++ .../module/filesystem/s3/utils/S3Utils.java | 24 ++++++++++++++ .../actions/FilesystemQueryActionTest.java | 15 +++++++-- .../s3/actions/S3QueryActionTest.java | 7 +++++ 5 files changed, 79 insertions(+), 6 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index e8ae33cf..df54eef9 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.filesystem.base.actions; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; @@ -52,6 +53,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinali import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.NotImplementedException; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -205,15 +207,17 @@ public abstract class AbstractBaseFilesystemAction AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); List files = listFiles(table, queryInput.getBackend(), queryInput.getFilter()); + int recordCount = 0; + + FILE_LOOP: for(FILE file : files) { - LOG.info("Processing file: " + getFullPathForFile(file)); - InputStream inputStream = readFile(file); switch(tableDetails.getCardinality()) { case MANY: { + LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file))); switch(tableDetails.getRecordFormat()) { case CSV: @@ -260,14 +264,33 @@ public abstract class AbstractBaseFilesystemAction } case ONE: { + //////////////////////////////////////////////////////////////////////////////// + // for one-record tables, put the entire file's contents into a single record // + //////////////////////////////////////////////////////////////////////////////// String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table); + byte[] bytes = inputStream.readAllBytes(); - byte[] bytes = inputStream.readAllBytes(); QRecord record = new QRecord() .withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase) .withValue(tableDetails.getContentsFieldName(), bytes); queryOutput.addRecord(record); + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // keep our own count - in case the query output is using a pipe (e.g., so we can't just call a .size()) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + recordCount++; + + //////////////////////////////////////////////////////////////////////////// + // break out of the file loop if we have hit the limit (if one was given) // + //////////////////////////////////////////////////////////////////////////// + if(queryInput.getFilter() != null && queryInput.getFilter().getLimit() != null) + { + if(recordCount >= queryInput.getFilter().getLimit()) + { + break FILE_LOOP; + } + } + break; } default: @@ -374,6 +397,8 @@ public abstract class AbstractBaseFilesystemAction QTableMetaData table = insertInput.getTable(); QBackendMetaData backend = insertInput.getBackend(); + output.setRecords(new ArrayList<>()); + AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); if(tableDetails.getCardinality().equals(Cardinality.ONE)) { diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index 8376d922..3e9f1919 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -85,6 +85,14 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction { if(filter != null && filter.hasAnyCriteria()) { + if(CollectionUtils.nullSafeHasContents(filter.getSubFilters())) + { + /////////////////////////////////////////////////////////////////////// + // todo - well, we could - just build up a tree of and's and or's... // + /////////////////////////////////////////////////////////////////////// + throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time.")); + } + List fileFilterList = new ArrayList<>(); for(QFilterCriteria criteria : filter.getCriteria()) { diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java index cfc0f9d4..c54e1d20 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java @@ -106,6 +106,10 @@ public class S3Utils useQQueryFilter = true; } + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if there's a filter for single file, make that file name the "prefix" that we send to s3, so we just get back that 1 file. // + // as this will be a common case. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// if(filter != null && useQQueryFilter) { if(filter.getCriteria() != null && filter.getCriteria().size() == 1) @@ -200,6 +204,18 @@ public class S3Utils } rs.add(objectSummary); + + ///////////////////////////////////////////////////////////////// + // if we have a limit, and we've hit it, break out of the loop // + ///////////////////////////////////////////////////////////////// + if(filter != null && useQQueryFilter && filter.getLimit() != null) + { + if(rs.size() >= filter.getLimit()) + { + break; + } + } + } } while(listObjectsV2Result.isTruncated()); @@ -219,6 +235,14 @@ public class S3Utils return (true); } + if(CollectionUtils.nullSafeHasContents(filter.getSubFilters())) + { + /////////////////////////////// + // todo - well, we could ... // + /////////////////////////////// + throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time.")); + } + Path path = Path.of(URI.create("file:///" + key)); //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java index 26d6629f..f8a636fc 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java @@ -93,13 +93,15 @@ public class FilesystemQueryActionTest extends FilesystemActionTest @Test public void testQueryForCardinalityOne() throws QException { + FilesystemQueryAction filesystemQueryAction = new FilesystemQueryAction(); + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS); queryInput.setFilter(new QQueryFilter()); - QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput); + QueryOutput queryOutput = filesystemQueryAction.execute(queryInput); assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); - queryOutput = new FilesystemQueryAction().execute(queryInput); + queryOutput = filesystemQueryAction.execute(queryInput); assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row"); assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName")); @@ -112,8 +114,15 @@ public class FilesystemQueryActionTest extends FilesystemActionTest reInitInstanceInContext(instance); queryInput.setFilter(new QQueryFilter()); - queryOutput = new FilesystemQueryAction().execute(queryInput); + queryOutput = filesystemQueryAction.execute(queryInput); assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows"); + + ////////////////////////////// + // add a limit to the query // + ////////////////////////////// + queryInput.setFilter(new QQueryFilter().withLimit(1)); + queryOutput = filesystemQueryAction.execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Query with limit should be respected"); } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java index 36a7c771..4e97044a 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java @@ -105,6 +105,13 @@ public class S3QueryActionTest extends BaseS3Test queryInput.setFilter(new QQueryFilter()); queryOutput = s3QueryAction.execute(queryInput); assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows"); + + ////////////////////////////// + // add a limit to the query // + ////////////////////////////// + queryInput.setFilter(new QQueryFilter().withLimit(1)); + queryOutput = s3QueryAction.execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Query with limit should be respected"); } } \ No newline at end of file From 2da6878e7029b56d83109bcec1a2fa028ae01ba9 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 28 Dec 2023 10:32:47 -0600 Subject: [PATCH 4/6] Make sure to always return an empty list rather than a null --- .../qqq/backend/core/actions/tables/InsertAction.java | 9 +++++++++ .../qqq/backend/core/actions/tables/UpdateAction.java | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index a537c883..9d60d611 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -132,6 +132,15 @@ public class InsertAction extends AbstractQActionFunction()); + } + ////////////////////////////// // log if there were errors // ////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java index a36decd0..69717f4f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java @@ -137,6 +137,15 @@ public class UpdateAction //////////////////////////////////// UpdateOutput updateOutput = updateInterface.execute(updateInput); + if(updateOutput.getRecords() == null) + { + //////////////////////////////////////////////////////////////////////////////////// + // in case the module failed to set record in the output, put an empty list there // + // to avoid so many downstream NPE's // + //////////////////////////////////////////////////////////////////////////////////// + updateOutput.setRecords(new ArrayList<>()); + } + ////////////////////////////// // log if there were errors // ////////////////////////////// From 872dec3177b413be64bc69ad1329f4ecb2743633 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 28 Dec 2023 16:38:40 -0600 Subject: [PATCH 5/6] CE-773 change fileNameFieldName and contentsFieldName to default as null - add validation to tableBackendDetails, specifically implemented in filesystem module --- .../core/instances/QInstanceValidator.java | 5 + .../metadata/tables/QTableBackendDetails.java | 15 ++ ...AbstractFilesystemTableBackendDetails.java | 48 ++++- .../backend/module/filesystem/TestUtils.java | 7 +- ...ractFilesystemTableBackendDetailsTest.java | 195 ++++++++++++++++++ 5 files changed, 262 insertions(+), 8 deletions(-) create mode 100644 qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetailsTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 4cd6bafd..725b9c4d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -480,6 +480,11 @@ public class QInstanceValidator validateTableCustomizer(tableName, entry.getKey(), entry.getValue()); } + if(table.getBackendDetails() != null) + { + table.getBackendDetails().validate(qInstance, table, this); + } + validateTableAutomationDetails(qInstance, table); validateTableUniqueKeys(table); validateAssociatedScripts(table); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableBackendDetails.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableBackendDetails.java index 77344ec0..732fa9e4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableBackendDetails.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableBackendDetails.java @@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.serialization.QTableBackendDetailsDeserializer; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -100,4 +103,16 @@ public abstract class QTableBackendDetails return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public void validate(QInstance qInstance, QTableMetaData table, QInstanceValidator qInstanceValidator) + { + //////////////////////// + // noop in base class // + //////////////////////// + } + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java index 95d4b438..f50f1b4c 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java @@ -22,7 +22,11 @@ package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -35,11 +39,9 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails private RecordFormat recordFormat; private Cardinality cardinality; - /////////////////////////////////////////////////////////////////////////////////////////////////// - // todo default these to null, and give validation error if not set for a cardinality=ONE table? // - /////////////////////////////////////////////////////////////////////////////////////////////////// - private String contentsFieldName = "contents"; - private String fileNameFieldName = "fileName"; + private String contentsFieldName; + private String fileNameFieldName; + /******************************************************************************* @@ -243,4 +245,40 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails } + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void validate(QInstance qInstance, QTableMetaData table, QInstanceValidator qInstanceValidator) + { + super.validate(qInstance, table, qInstanceValidator); + + String prefix = "Table " + (table == null ? "null" : table.getName()) + " backend details - "; + if(qInstanceValidator.assertCondition(cardinality != null, prefix + "missing cardinality")) + { + if(cardinality.equals(Cardinality.ONE)) + { + if(qInstanceValidator.assertCondition(StringUtils.hasContent(contentsFieldName), prefix + "missing contentsFieldName, which is required for Cardinality ONE")) + { + qInstanceValidator.assertCondition(table != null && table.getFields().containsKey(contentsFieldName), prefix + "contentsFieldName [" + contentsFieldName + "] is not a field on this table."); + } + + if(qInstanceValidator.assertCondition(StringUtils.hasContent(fileNameFieldName), prefix + "missing fileNameFieldName, which is required for Cardinality ONE")) + { + qInstanceValidator.assertCondition(table != null && table.getFields().containsKey(fileNameFieldName), prefix + "fileNameFieldName [" + fileNameFieldName + "] is not a field on this table."); + } + + qInstanceValidator.assertCondition(recordFormat == null, prefix + "has a recordFormat, which is not allowed for Cardinality ONE"); + } + + if(cardinality.equals(Cardinality.MANY)) + { + qInstanceValidator.assertCondition(!StringUtils.hasContent(contentsFieldName), prefix + "has a contentsFieldName, which is not allowed for Cardinality MANY"); + qInstanceValidator.assertCondition(!StringUtils.hasContent(fileNameFieldName), prefix + "has a fileNameFieldName, which is not allowed for Cardinality MANY"); + qInstanceValidator.assertCondition(recordFormat != null, prefix + "missing recordFormat, which is required for Cardinality MANY"); + } + } + + } } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index c209f6a5..4510ecd5 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -26,7 +26,6 @@ import java.io.File; import java.io.IOException; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -142,8 +141,6 @@ public class TestUtils qInstance.addTable(defineMockPersonTable()); qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess()); - new QInstanceValidator().validate(qInstance); - return (qInstance); } @@ -249,6 +246,8 @@ public class TestUtils .withBackendDetails(new FilesystemTableBackendDetails() .withBasePath("blobs") .withCardinality(Cardinality.ONE) + .withFileNameFieldName("fileName") + .withContentsFieldName("contents") ); } @@ -269,6 +268,8 @@ public class TestUtils .withBackendDetails(new S3TableBackendDetails() .withBasePath("blobs") .withCardinality(Cardinality.ONE) + .withFileNameFieldName("fileName") + .withContentsFieldName("contents") ); } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetailsTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetailsTest.java new file mode 100644 index 00000000..7fc78997 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetailsTest.java @@ -0,0 +1,195 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; + + +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.module.filesystem.BaseTest; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** Unit test for AbstractFilesystemTableBackendDetails + *******************************************************************************/ +class AbstractFilesystemTableBackendDetailsTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValidInstancePasses() throws QInstanceValidationException + { + new QInstanceValidator().validate(QContext.getQInstance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testMissingCardinality() throws QException + { + assertValidationFailureReasons((QInstance qInstance) -> + { + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3).withBackendDetails(new FilesystemTableBackendDetails()); + }, false, "missing cardinality"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCardinalityOneIssues() throws QException + { + assertValidationFailureReasons((QInstance qInstance) -> + { + qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withBackendDetails(new FilesystemTableBackendDetails() + .withCardinality(Cardinality.ONE) + ); + }, false, "missing contentsFieldName", "missing fileNameFieldName"); + + assertValidationFailureReasons((QInstance qInstance) -> + { + qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withBackendDetails(new FilesystemTableBackendDetails() + .withCardinality(Cardinality.ONE) + .withContentsFieldName("foo") + .withFileNameFieldName("bar") + ); + }, false, "contentsFieldName [foo] is not a field", "fileNameFieldName [bar] is not a field"); + + assertValidationFailureReasons((QInstance qInstance) -> + { + qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withBackendDetails(new FilesystemTableBackendDetails() + .withCardinality(Cardinality.ONE) + .withContentsFieldName("contents") + .withFileNameFieldName("fileName") + .withRecordFormat(RecordFormat.CSV) + ); + }, false, "has a recordFormat"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCardinalityManyIssues() throws QException + { + assertValidationFailureReasons((QInstance qInstance) -> + { + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_CSV).withBackendDetails(new FilesystemTableBackendDetails() + .withCardinality(Cardinality.MANY) + ); + }, false, "missing recordFormat"); + + assertValidationFailureReasons((QInstance qInstance) -> + { + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_CSV).withBackendDetails(new FilesystemTableBackendDetails() + .withCardinality(Cardinality.MANY) + .withRecordFormat(RecordFormat.CSV) + .withContentsFieldName("foo") + .withFileNameFieldName("bar") + ); + }, false, "has a contentsFieldName", "has a fileNameFieldName"); + } + + + + /******************************************************************************* + ** Implementation for the overloads of this name. + *******************************************************************************/ + private void assertValidationFailureReasons(Consumer setup, boolean allowExtraReasons, String... reasons) throws QException + { + try + { + QInstance qInstance = TestUtils.defineInstance(); + setup.accept(qInstance); + new QInstanceValidator().validate(qInstance); + fail("Should have thrown validationException"); + } + catch(QInstanceValidationException e) + { + if(!allowExtraReasons) + { + int noOfReasons = e.getReasons() == null ? 0 : e.getReasons().size(); + assertEquals(reasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", reasons) + + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", e.getReasons()) : "--")); + } + + for(String reason : reasons) + { + assertReason(reason, e); + } + } + } + + + + /******************************************************************************* + ** Assert that an instance is valid! + *******************************************************************************/ + private void assertValidationSuccess(Consumer setup) throws QException + { + try + { + QInstance qInstance = TestUtils.defineInstance(); + setup.accept(qInstance); + new QInstanceValidator().validate(qInstance); + } + catch(QInstanceValidationException e) + { + fail("Expected no validation errors, but received: " + e.getMessage()); + } + } + + + + /******************************************************************************* + ** utility method for asserting that a specific reason string is found within + ** the list of reasons in the QInstanceValidationException. + ** + *******************************************************************************/ + private void assertReason(String reason, QInstanceValidationException e) + { + assertNotNull(e.getReasons(), "Expected there to be a reason for the failure (but there was not)"); + assertThat(e.getReasons()) + .withFailMessage("Expected any of:\n%s\nTo match: [%s]", e.getReasons(), reason) + .anyMatch(s -> s.contains(reason)); + } + +} \ No newline at end of file From 6e1ea5c8f1f51caa04644416eb2411817679ada4 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 28 Dec 2023 16:46:51 -0600 Subject: [PATCH 6/6] CE-773 fix tables created in here, per new validationing! --- .../filesystem/sync/FilesystemSyncProcessS3Test.java | 8 ++++++-- .../filesystem/sync/FilesystemSyncProcessTest.java | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java index 1ef2f8d9..675f593f 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java @@ -38,6 +38,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModuleSubclassForTest; @@ -197,8 +199,8 @@ class FilesystemSyncProcessS3Test extends BaseS3Test for(String path : paths) { assertTrue(s3ObjectSummaries.stream().anyMatch(s3o -> s3o.getKey().equals(path)), - "Path [" + path + "] should be in the listing, but was not. Full listing is: " + - s3ObjectSummaries.stream().map(S3ObjectSummary::getKey).collect(Collectors.joining(","))); + "Path [" + path + "] should be in the listing, but was not. Full listing is: " + + s3ObjectSummaries.stream().map(S3ObjectSummary::getKey).collect(Collectors.joining(","))); } } @@ -257,6 +259,8 @@ class FilesystemSyncProcessS3Test extends BaseS3Test .withBackendName(backend.getName()) .withField(new QFieldMetaData("id", QFieldType.INTEGER)) .withBackendDetails(new S3TableBackendDetails() + .withCardinality(Cardinality.MANY) + .withRecordFormat(RecordFormat.CSV) .withBasePath(path) .withGlob(glob)); qInstance.addTable(qTableMetaData); diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java index 9ff75ce7..0f38e654 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java @@ -35,6 +35,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.module.filesystem.BaseTest; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import org.apache.commons.io.FileUtils; @@ -118,6 +120,8 @@ class FilesystemSyncProcessTest extends BaseTest .withBackendName(TestUtils.BACKEND_NAME_LOCAL_FS) .withField(new QFieldMetaData("id", QFieldType.INTEGER)) .withBackendDetails(new FilesystemTableBackendDetails() + .withCardinality(Cardinality.MANY) + .withRecordFormat(RecordFormat.CSV) .withBasePath(subPath)); }