mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 05:01:07 +00:00
CE-773 update to support listing/filtering filesystem tables with Cardinality.ONE (single-record per-file)
This commit is contained in:
@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
|||||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
|
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
|
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
|
||||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||||
@ -68,7 +69,17 @@ public abstract class AbstractBaseFilesystemAction<FILE>
|
|||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
** List the files for a table - to be implemented in module-specific subclasses.
|
** List the files for a table - to be implemented in module-specific subclasses.
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase);
|
public List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase) throws QException
|
||||||
|
{
|
||||||
|
return (listFiles(table, backendBase, null));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses.
|
||||||
|
*******************************************************************************/
|
||||||
|
public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException;
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
** Read the contents of a file - to be implemented in module-specific subclasses.
|
** Read the contents of a file - to be implemented in module-specific subclasses.
|
||||||
@ -181,6 +192,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
|
|||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
** Generic implementation of the execute method from the QueryInterface
|
** Generic implementation of the execute method from the QueryInterface
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
|
@SuppressWarnings("checkstyle:Indentation")
|
||||||
public QueryOutput executeQuery(QueryInput queryInput) throws QException
|
public QueryOutput executeQuery(QueryInput queryInput) throws QException
|
||||||
{
|
{
|
||||||
preAction(queryInput.getBackend());
|
preAction(queryInput.getBackend());
|
||||||
@ -191,51 +203,76 @@ public abstract class AbstractBaseFilesystemAction<FILE>
|
|||||||
|
|
||||||
QTableMetaData table = queryInput.getTable();
|
QTableMetaData table = queryInput.getTable();
|
||||||
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
|
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
|
||||||
List<FILE> files = listFiles(table, queryInput.getBackend());
|
List<FILE> files = listFiles(table, queryInput.getBackend(), queryInput.getFilter());
|
||||||
|
|
||||||
for(FILE file : files)
|
for(FILE file : files)
|
||||||
{
|
{
|
||||||
LOG.info("Processing file: " + getFullPathForFile(file));
|
LOG.info("Processing file: " + getFullPathForFile(file));
|
||||||
switch(tableDetails.getRecordFormat())
|
|
||||||
{
|
|
||||||
case CSV:
|
|
||||||
{
|
|
||||||
String fileContents = IOUtils.toString(readFile(file));
|
|
||||||
fileContents = customizeFileContentsAfterReading(table, fileContents);
|
|
||||||
|
|
||||||
if(queryInput.getRecordPipe() != null)
|
InputStream inputStream = readFile(file);
|
||||||
|
switch(tableDetails.getCardinality())
|
||||||
|
{
|
||||||
|
case MANY:
|
||||||
|
{
|
||||||
|
switch(tableDetails.getRecordFormat())
|
||||||
{
|
{
|
||||||
new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record ->
|
case CSV:
|
||||||
{
|
{
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////
|
String fileContents = IOUtils.toString(inputStream);
|
||||||
// Before the records go into the pipe, make sure their backend details are added to them //
|
fileContents = customizeFileContentsAfterReading(table, fileContents);
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
addBackendDetailsToRecord(record, file);
|
if(queryInput.getRecordPipe() != null)
|
||||||
}));
|
{
|
||||||
}
|
new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record ->
|
||||||
else
|
{
|
||||||
{
|
////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
|
// Before the records go into the pipe, make sure their backend details are added to them //
|
||||||
addBackendDetailsToRecords(recordsInFile, file);
|
////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
queryOutput.addRecords(recordsInFile);
|
addBackendDetailsToRecord(record, file);
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
|
||||||
|
addBackendDetailsToRecords(recordsInFile, file);
|
||||||
|
queryOutput.addRecords(recordsInFile);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case JSON:
|
||||||
|
{
|
||||||
|
String fileContents = IOUtils.toString(inputStream);
|
||||||
|
fileContents = customizeFileContentsAfterReading(table, fileContents);
|
||||||
|
|
||||||
|
// todo - pipe support!!
|
||||||
|
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
|
||||||
|
addBackendDetailsToRecords(recordsInFile, file);
|
||||||
|
|
||||||
|
queryOutput.addRecords(recordsInFile);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
{
|
||||||
|
throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case JSON:
|
case ONE:
|
||||||
{
|
{
|
||||||
String fileContents = IOUtils.toString(readFile(file));
|
String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table);
|
||||||
fileContents = customizeFileContentsAfterReading(table, fileContents);
|
|
||||||
|
|
||||||
// todo - pipe support!!
|
byte[] bytes = inputStream.readAllBytes();
|
||||||
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
|
QRecord record = new QRecord()
|
||||||
addBackendDetailsToRecords(recordsInFile, file);
|
.withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase)
|
||||||
|
.withValue(tableDetails.getContentsFieldName(), bytes);
|
||||||
|
queryOutput.addRecord(record);
|
||||||
|
|
||||||
queryOutput.addRecords(recordsInFile);
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
{
|
{
|
||||||
throw new NotImplementedException("Filesystem record format " + tableDetails.getRecordFormat() + " is not yet implemented");
|
throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -342,8 +379,8 @@ public abstract class AbstractBaseFilesystemAction<FILE>
|
|||||||
{
|
{
|
||||||
for(QRecord record : insertInput.getRecords())
|
for(QRecord record : insertInput.getRecords())
|
||||||
{
|
{
|
||||||
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString("fileName"));
|
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
|
||||||
writeFile(backend, fullPath, record.getValueByteArray("contents"));
|
writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
|
||||||
record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
|
record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
|
||||||
output.addRecord(record);
|
output.addRecord(record);
|
||||||
}
|
}
|
||||||
|
@ -35,6 +35,11 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
|
|||||||
private RecordFormat recordFormat;
|
private RecordFormat recordFormat;
|
||||||
private Cardinality cardinality;
|
private Cardinality cardinality;
|
||||||
|
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// todo default these to null, and give validation error if not set for a cardinality=ONE table? //
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
private String contentsFieldName = "contents";
|
||||||
|
private String fileNameFieldName = "fileName";
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
@ -175,4 +180,67 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
|
|||||||
return ((T) this);
|
return ((T) this);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Getter for contentsFieldName
|
||||||
|
*******************************************************************************/
|
||||||
|
public String getContentsFieldName()
|
||||||
|
{
|
||||||
|
return (this.contentsFieldName);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Setter for contentsFieldName
|
||||||
|
*******************************************************************************/
|
||||||
|
public void setContentsFieldName(String contentsFieldName)
|
||||||
|
{
|
||||||
|
this.contentsFieldName = contentsFieldName;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Fluent setter for contentsFieldName
|
||||||
|
*******************************************************************************/
|
||||||
|
public AbstractFilesystemTableBackendDetails withContentsFieldName(String contentsFieldName)
|
||||||
|
{
|
||||||
|
this.contentsFieldName = contentsFieldName;
|
||||||
|
return (this);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Getter for fileNameFieldName
|
||||||
|
*******************************************************************************/
|
||||||
|
public String getFileNameFieldName()
|
||||||
|
{
|
||||||
|
return (this.fileNameFieldName);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Setter for fileNameFieldName
|
||||||
|
*******************************************************************************/
|
||||||
|
public void setFileNameFieldName(String fileNameFieldName)
|
||||||
|
{
|
||||||
|
this.fileNameFieldName = fileNameFieldName;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Fluent setter for fileNameFieldName
|
||||||
|
*******************************************************************************/
|
||||||
|
public AbstractFilesystemTableBackendDetails withFileNameFieldName(String fileNameFieldName)
|
||||||
|
{
|
||||||
|
this.fileNameFieldName = fileNameFieldName;
|
||||||
|
return (this);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -23,19 +23,36 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
|||||||
|
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
|
import java.io.FileFilter;
|
||||||
import java.io.FileInputStream;
|
import java.io.FileInputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||||
|
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||||
|
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
|
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
|
||||||
|
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
||||||
|
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
|
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
|
||||||
import org.apache.commons.io.FileUtils;
|
import org.apache.commons.io.FileUtils;
|
||||||
|
import org.apache.commons.io.IOCase;
|
||||||
|
import org.apache.commons.io.filefilter.AndFileFilter;
|
||||||
|
import org.apache.commons.io.filefilter.NameFileFilter;
|
||||||
|
import org.apache.commons.io.filefilter.OrFileFilter;
|
||||||
|
import org.apache.commons.io.filefilter.TrueFileFilter;
|
||||||
|
import org.apache.commons.io.filefilter.WildcardFileFilter;
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
@ -51,12 +68,66 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
|
|||||||
** List the files for this table.
|
** List the files for this table.
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
@Override
|
@Override
|
||||||
public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase)
|
public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
|
||||||
{
|
{
|
||||||
// todo - needs rewritten to do globbing...
|
|
||||||
String fullPath = getFullBasePath(table, backendBase);
|
String fullPath = getFullBasePath(table, backendBase);
|
||||||
File directory = new File(fullPath);
|
File directory = new File(fullPath);
|
||||||
File[] files = directory.listFiles();
|
File[] files = null;
|
||||||
|
|
||||||
|
AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
|
||||||
|
|
||||||
|
FileFilter fileFilter = TrueFileFilter.INSTANCE;
|
||||||
|
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// if each file is its own record (ONE), then we may need to do filtering of the directory listing based on the input filter //
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
if(Cardinality.ONE.equals(tableBackendDetails.getCardinality()))
|
||||||
|
{
|
||||||
|
if(filter != null && filter.hasAnyCriteria())
|
||||||
|
{
|
||||||
|
List<FileFilter> fileFilterList = new ArrayList<>();
|
||||||
|
for(QFilterCriteria criteria : filter.getCriteria())
|
||||||
|
{
|
||||||
|
if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
|
||||||
|
{
|
||||||
|
if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
|
||||||
|
{
|
||||||
|
fileFilterList.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(0))));
|
||||||
|
}
|
||||||
|
else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
|
||||||
|
{
|
||||||
|
List<NameFileFilter> nameInFilters = new ArrayList<>();
|
||||||
|
for(int i = 0; i < criteria.getValues().size(); i++)
|
||||||
|
{
|
||||||
|
nameInFilters.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(i))));
|
||||||
|
}
|
||||||
|
fileFilterList.add(new OrFileFilter(nameInFilters));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fileFilter = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? new AndFileFilter(fileFilterList) : new OrFileFilter(fileFilterList);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// if the table has a glob specified, add it as an AND to the filter built to this point //
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
if(StringUtils.hasContent(tableBackendDetails.getGlob()))
|
||||||
|
{
|
||||||
|
WildcardFileFilter globFilenameFilter = new WildcardFileFilter(tableBackendDetails.getGlob(), IOCase.INSENSITIVE);
|
||||||
|
fileFilter = new AndFileFilter(List.of(globFilenameFilter, fileFilter));
|
||||||
|
}
|
||||||
|
|
||||||
|
files = directory.listFiles(fileFilter);
|
||||||
|
|
||||||
if(files == null)
|
if(files == null)
|
||||||
{
|
{
|
||||||
|
@ -30,7 +30,9 @@ import com.amazonaws.auth.BasicAWSCredentials;
|
|||||||
import com.amazonaws.services.s3.AmazonS3;
|
import com.amazonaws.services.s3.AmazonS3;
|
||||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||||
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||||
@ -126,7 +128,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
|
|||||||
** List the files for a table.
|
** List the files for a table.
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
@Override
|
@Override
|
||||||
public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase)
|
public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
|
||||||
{
|
{
|
||||||
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase);
|
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase);
|
||||||
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
|
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
|
||||||
@ -138,7 +140,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
|
|||||||
////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////
|
||||||
// todo - look at metadata to configure the s3 client here? //
|
// todo - look at metadata to configure the s3 client here? //
|
||||||
////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////
|
||||||
return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob);
|
return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob, filter, tableDetails);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -37,7 +37,14 @@ import com.amazonaws.services.s3.model.ListObjectsV2Request;
|
|||||||
import com.amazonaws.services.s3.model.ListObjectsV2Result;
|
import com.amazonaws.services.s3.model.ListObjectsV2Result;
|
||||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||||
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||||
|
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||||
|
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
||||||
|
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
|
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
|
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
|
||||||
|
|
||||||
@ -61,7 +68,19 @@ public class S3Utils
|
|||||||
** List the objects in an S3 bucket matching a glob, per:
|
** List the objects in an S3 bucket matching a glob, per:
|
||||||
** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
|
** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob)
|
public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob) throws QException
|
||||||
|
{
|
||||||
|
return listObjectsInBucketMatchingGlob(bucketName, path, glob, null, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** List the objects in an S3 bucket matching a glob, per:
|
||||||
|
** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
|
||||||
|
** and also - (possibly) apply a file-name filter (based on the table's details).
|
||||||
|
*******************************************************************************/
|
||||||
|
public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
|
||||||
{
|
{
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
// s3 list requests find nothing if the path starts with a /, so strip away any leading slashes //
|
// s3 list requests find nothing if the path starts with a /, so strip away any leading slashes //
|
||||||
@ -77,6 +96,28 @@ public class S3Utils
|
|||||||
prefix = prefix.substring(0, prefix.indexOf('*'));
|
prefix = prefix.substring(0, prefix.indexOf('*'));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// for a file-per-record (ONE) table, we may need to apply the filter to listing. //
|
||||||
|
// but for MANY tables, the filtering would be done on the records after they came out of the files. //
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
boolean useQQueryFilter = false;
|
||||||
|
if(tableDetails != null && Cardinality.ONE.equals(tableDetails.getCardinality()))
|
||||||
|
{
|
||||||
|
useQQueryFilter = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(filter != null && useQQueryFilter)
|
||||||
|
{
|
||||||
|
if(filter.getCriteria() != null && filter.getCriteria().size() == 1)
|
||||||
|
{
|
||||||
|
QFilterCriteria criteria = filter.getCriteria().get(0);
|
||||||
|
if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS))
|
||||||
|
{
|
||||||
|
prefix += "/" + criteria.getValues().get(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
// mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob. //
|
// mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob. //
|
||||||
// That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. //
|
// That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. //
|
||||||
@ -86,6 +127,7 @@ public class S3Utils
|
|||||||
{
|
{
|
||||||
glob = "";
|
glob = "";
|
||||||
}
|
}
|
||||||
|
|
||||||
if(!glob.contains("*"))
|
if(!glob.contains("*"))
|
||||||
{
|
{
|
||||||
if(glob.equals(""))
|
if(glob.equals(""))
|
||||||
@ -114,7 +156,7 @@ public class S3Utils
|
|||||||
{
|
{
|
||||||
listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken());
|
listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken());
|
||||||
}
|
}
|
||||||
LOG.info("Listing bucket=" + bucketName + ", path=" + path);
|
LOG.info("Listing bucket=" + bucketName + ", path=" + path + ", prefix=" + prefix + ", glob=" + glob);
|
||||||
listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request);
|
listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request);
|
||||||
|
|
||||||
//////////////////////////////////
|
//////////////////////////////////
|
||||||
@ -149,6 +191,14 @@ public class S3Utils
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// if we're a file-per-record table, and we have a filter, compare the key to it //
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////
|
||||||
|
if(!doesObjectKeyMatchFilter(key, filter, tableDetails))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
rs.add(objectSummary);
|
rs.add(objectSummary);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -159,6 +209,95 @@ public class S3Utils
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
private boolean doesObjectKeyMatchFilter(String key, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
|
||||||
|
{
|
||||||
|
if(filter == null || !filter.hasAnyCriteria())
|
||||||
|
{
|
||||||
|
return (true);
|
||||||
|
}
|
||||||
|
|
||||||
|
Path path = Path.of(URI.create("file:///" + key));
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches //
|
||||||
|
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
for(QFilterCriteria criteria : filter.getCriteria())
|
||||||
|
{
|
||||||
|
boolean matches = doesObjectKeyMatchOneCriteria(criteria, tableDetails, path);
|
||||||
|
|
||||||
|
if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
|
||||||
|
{
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// if it's not a match, and it's an AND filter, then the whole thing is false //
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
return (false);
|
||||||
|
}
|
||||||
|
|
||||||
|
if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
|
||||||
|
{
|
||||||
|
////////////////////////////////////////////////////////////
|
||||||
|
// if it's an OR filter, and we've a match, return a true //
|
||||||
|
////////////////////////////////////////////////////////////
|
||||||
|
return (true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////////////////////
|
||||||
|
// if we didn't return above, return now //
|
||||||
|
// for an OR - if we didn't find something true, then return false. //
|
||||||
|
// else, an AND - if we didn't find a false, we can return true. //
|
||||||
|
//////////////////////////////////////////////////////////////////////
|
||||||
|
if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
|
||||||
|
{
|
||||||
|
return (false);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (true);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
private static boolean doesObjectKeyMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException
|
||||||
|
{
|
||||||
|
if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
|
||||||
|
{
|
||||||
|
if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
|
||||||
|
{
|
||||||
|
return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path));
|
||||||
|
}
|
||||||
|
else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
|
||||||
|
{
|
||||||
|
boolean anyMatch = false;
|
||||||
|
for(int i = 0; i < criteria.getValues().size(); i++)
|
||||||
|
{
|
||||||
|
if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path))
|
||||||
|
{
|
||||||
|
anyMatch = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return (anyMatch);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
** Get the contents (as an InputStream) for an object in s3
|
** Get the contents (as an InputStream) for an object in s3
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
@ -245,4 +384,5 @@ public class S3Utils
|
|||||||
|
|
||||||
return amazonS3;
|
return amazonS3;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -25,6 +25,11 @@ package com.kingsrook.qqq.backend.module.filesystem.local;
|
|||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||||
@ -36,6 +41,8 @@ import org.junit.jupiter.api.AfterEach;
|
|||||||
import org.junit.jupiter.api.Assertions;
|
import org.junit.jupiter.api.Assertions;
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
@ -47,6 +54,9 @@ public class FilesystemBackendModuleTest
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
@BeforeEach
|
@BeforeEach
|
||||||
public void beforeEach() throws IOException
|
public void beforeEach() throws IOException
|
||||||
{
|
{
|
||||||
@ -55,6 +65,9 @@ public class FilesystemBackendModuleTest
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
@AfterEach
|
@AfterEach
|
||||||
public void afterEach() throws Exception
|
public void afterEach() throws Exception
|
||||||
{
|
{
|
||||||
@ -63,6 +76,78 @@ public class FilesystemBackendModuleTest
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
void testListFiles() throws QException
|
||||||
|
{
|
||||||
|
QInstance qInstance = TestUtils.defineInstance();
|
||||||
|
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
|
||||||
|
|
||||||
|
AbstractFilesystemAction abstractFilesystemAction = new AbstractFilesystemAction();
|
||||||
|
QBackendMetaData backend = qInstance.getBackendForTable(table.getName());
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////////
|
||||||
|
// with no filter given, all (3) files should come back //
|
||||||
|
//////////////////////////////////////////////////////////
|
||||||
|
List<File> files = abstractFilesystemAction.listFiles(table, backend);
|
||||||
|
assertEquals(3, files.size());
|
||||||
|
|
||||||
|
/////////////////////////////////////////
|
||||||
|
// filter for a file name that's found //
|
||||||
|
/////////////////////////////////////////
|
||||||
|
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
|
||||||
|
assertEquals(1, files.size());
|
||||||
|
assertEquals("BLOB-2.txt", files.get(0).getName());
|
||||||
|
|
||||||
|
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
|
||||||
|
assertEquals(1, files.size());
|
||||||
|
assertEquals("BLOB-1.txt", files.get(0).getName());
|
||||||
|
|
||||||
|
///////////////////////////////////
|
||||||
|
// filter for 2 names that exist //
|
||||||
|
///////////////////////////////////
|
||||||
|
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
|
||||||
|
assertEquals(2, files.size());
|
||||||
|
|
||||||
|
/////////////////////////////////////////////
|
||||||
|
// filter for a file name that isn't found //
|
||||||
|
/////////////////////////////////////////////
|
||||||
|
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
|
||||||
|
assertEquals(0, files.size());
|
||||||
|
|
||||||
|
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
|
||||||
|
assertEquals(1, files.size());
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////
|
||||||
|
// 2 criteria, and'ed, and can't match, so find 0 //
|
||||||
|
////////////////////////////////////////////////////
|
||||||
|
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
|
||||||
|
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
|
||||||
|
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
|
||||||
|
assertEquals(0, files.size());
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////
|
||||||
|
// 2 criteria, or'ed, and both match, so find 2 //
|
||||||
|
//////////////////////////////////////////////////
|
||||||
|
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
|
||||||
|
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
|
||||||
|
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
|
||||||
|
.withBooleanOperator(QQueryFilter.BooleanOperator.OR));
|
||||||
|
assertEquals(2, files.size());
|
||||||
|
|
||||||
|
//////////////////////////////////////
|
||||||
|
// ensure unsupported filters throw //
|
||||||
|
//////////////////////////////////////
|
||||||
|
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
|
||||||
|
.hasMessageContaining("Unable to query filesystem table by field");
|
||||||
|
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
|
||||||
|
.hasMessageContaining("Unable to query filename field using operator");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
**
|
**
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
|
@ -89,6 +89,7 @@ public class FilesystemActionTest extends BaseTest
|
|||||||
|
|
||||||
writePersonJSONFiles(baseDirectory);
|
writePersonJSONFiles(baseDirectory);
|
||||||
writePersonCSVFiles(baseDirectory);
|
writePersonCSVFiles(baseDirectory);
|
||||||
|
writeBlobFiles(baseDirectory);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -158,6 +159,37 @@ public class FilesystemActionTest extends BaseTest
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Write some data files into the directory for the filesystem module.
|
||||||
|
*******************************************************************************/
|
||||||
|
private void writeBlobFiles(File baseDirectory) throws IOException
|
||||||
|
{
|
||||||
|
String fullPath = baseDirectory.getAbsolutePath();
|
||||||
|
if(TestUtils.defineLocalFilesystemBlobTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
|
||||||
|
{
|
||||||
|
if(StringUtils.hasContent(details.getBasePath()))
|
||||||
|
{
|
||||||
|
fullPath += File.separatorChar + details.getBasePath();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fullPath += File.separatorChar;
|
||||||
|
|
||||||
|
String data1 = """
|
||||||
|
Hello, Blob
|
||||||
|
""";
|
||||||
|
FileUtils.writeStringToFile(new File(fullPath + "BLOB-1.txt"), data1);
|
||||||
|
|
||||||
|
String data2 = """
|
||||||
|
Hi Bob""";
|
||||||
|
FileUtils.writeStringToFile(new File(fullPath + "BLOB-2.txt"), data2);
|
||||||
|
|
||||||
|
String data3 = """
|
||||||
|
# Hi MD...""";
|
||||||
|
FileUtils.writeStringToFile(new File(fullPath + "BLOB-3.md"), data3);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
**
|
**
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
|
@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
|||||||
|
|
||||||
|
|
||||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||||
@ -32,8 +35,10 @@ import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
|||||||
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
|
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractPostReadFileCustomizer;
|
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractPostReadFileCustomizer;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.base.actions.FilesystemTableCustomizers;
|
import com.kingsrook.qqq.backend.module.filesystem.base.actions.FilesystemTableCustomizers;
|
||||||
import org.junit.jupiter.api.Assertions;
|
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
@ -51,8 +56,8 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
|
|||||||
QueryInput queryInput = new QueryInput();
|
QueryInput queryInput = new QueryInput();
|
||||||
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
|
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
|
||||||
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
|
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
|
||||||
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
|
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
|
||||||
Assertions.assertTrue(queryOutput.getRecords().stream()
|
assertTrue(queryOutput.getRecords().stream()
|
||||||
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
|
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
|
||||||
"All records should have a full-path in their backend details, matching the test folder name");
|
"All records should have a full-path in their backend details, matching the test folder name");
|
||||||
}
|
}
|
||||||
@ -74,14 +79,48 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
|
|||||||
|
|
||||||
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
|
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
|
||||||
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
|
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
|
||||||
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
|
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
|
||||||
Assertions.assertTrue(
|
assertTrue(
|
||||||
queryOutput.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")),
|
queryOutput.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")),
|
||||||
"All records should have their email addresses up-shifted.");
|
"All records should have their email addresses up-shifted.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
public void testQueryForCardinalityOne() throws QException
|
||||||
|
{
|
||||||
|
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
|
||||||
|
queryInput.setFilter(new QQueryFilter());
|
||||||
|
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
|
||||||
|
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
|
||||||
|
|
||||||
|
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
|
||||||
|
queryOutput = new FilesystemQueryAction().execute(queryInput);
|
||||||
|
assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
|
||||||
|
assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////
|
||||||
|
// put a glob on the table - now should only find 2 txt files //
|
||||||
|
////////////////////////////////////////////////////////////////
|
||||||
|
QInstance instance = TestUtils.defineInstance();
|
||||||
|
((FilesystemTableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).getBackendDetails()))
|
||||||
|
.withGlob("*.txt");
|
||||||
|
reInitInstanceInContext(instance);
|
||||||
|
|
||||||
|
queryInput.setFilter(new QQueryFilter());
|
||||||
|
queryOutput = new FilesystemQueryAction().execute(queryInput);
|
||||||
|
assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
public static class ValueUpshifter extends AbstractPostReadFileCustomizer
|
public static class ValueUpshifter extends AbstractPostReadFileCustomizer
|
||||||
{
|
{
|
||||||
@Override
|
@Override
|
||||||
|
@ -26,7 +26,7 @@ import java.util.List;
|
|||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||||
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
|
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
|
||||||
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||||
@ -187,7 +187,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
|
|||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
**
|
**
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException
|
private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QException
|
||||||
{
|
{
|
||||||
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
|
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
|
||||||
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
|
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
|
||||||
@ -207,7 +207,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
|
|||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
**
|
**
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException
|
private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QException
|
||||||
{
|
{
|
||||||
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
|
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
|
||||||
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
|
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
|
||||||
|
@ -62,6 +62,9 @@ public class BaseS3Test extends BaseTest
|
|||||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2());
|
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2());
|
||||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test");
|
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test");
|
||||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
|
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
|
||||||
|
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-1.txt", "Hello, Blob");
|
||||||
|
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-2.txt", "Hi, Bob");
|
||||||
|
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-3.md", "# Hi, MD");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -25,6 +25,11 @@ package com.kingsrook.qqq.backend.module.filesystem.s3;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||||
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||||
@ -32,6 +37,9 @@ import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemExceptio
|
|||||||
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
|
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
|
||||||
import org.junit.jupiter.api.Assertions;
|
import org.junit.jupiter.api.Assertions;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
|
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
@ -43,6 +51,83 @@ public class S3BackendModuleTest extends BaseS3Test
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
void testListFiles() throws QException
|
||||||
|
{
|
||||||
|
QInstance qInstance = TestUtils.defineInstance();
|
||||||
|
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_S3);
|
||||||
|
QBackendMetaData backend = qInstance.getBackendForTable(table.getName());
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////
|
||||||
|
// set up the backend module (e.g., for localstack) //
|
||||||
|
//////////////////////////////////////////////////////
|
||||||
|
S3BackendModule s3BackendModule = new S3BackendModule();
|
||||||
|
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
|
||||||
|
actionBase.setS3Utils(getS3Utils());
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////////
|
||||||
|
// with no filter given, all (3) files should come back //
|
||||||
|
//////////////////////////////////////////////////////////
|
||||||
|
List<S3ObjectSummary> files = actionBase.listFiles(table, backend);
|
||||||
|
assertEquals(3, files.size());
|
||||||
|
|
||||||
|
/////////////////////////////////////////
|
||||||
|
// filter for a file name that's found //
|
||||||
|
/////////////////////////////////////////
|
||||||
|
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
|
||||||
|
assertEquals(1, files.size());
|
||||||
|
assertThat(files.get(0).getKey()).contains("BLOB-2.txt");
|
||||||
|
|
||||||
|
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
|
||||||
|
assertEquals(1, files.size());
|
||||||
|
assertThat(files.get(0).getKey()).contains("BLOB-1.txt");
|
||||||
|
|
||||||
|
///////////////////////////////////
|
||||||
|
// filter for 2 names that exist //
|
||||||
|
///////////////////////////////////
|
||||||
|
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
|
||||||
|
assertEquals(2, files.size());
|
||||||
|
|
||||||
|
/////////////////////////////////////////////
|
||||||
|
// filter for a file name that isn't found //
|
||||||
|
/////////////////////////////////////////////
|
||||||
|
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
|
||||||
|
assertEquals(0, files.size());
|
||||||
|
|
||||||
|
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
|
||||||
|
assertEquals(1, files.size());
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////
|
||||||
|
// 2 criteria, and'ed, and can't match, so find 0 //
|
||||||
|
////////////////////////////////////////////////////
|
||||||
|
files = actionBase.listFiles(table, backend, new QQueryFilter(
|
||||||
|
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
|
||||||
|
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
|
||||||
|
assertEquals(0, files.size());
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////
|
||||||
|
// 2 criteria, or'ed, and both match, so find 2 //
|
||||||
|
//////////////////////////////////////////////////
|
||||||
|
files = actionBase.listFiles(table, backend, new QQueryFilter(
|
||||||
|
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
|
||||||
|
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
|
||||||
|
.withBooleanOperator(QQueryFilter.BooleanOperator.OR));
|
||||||
|
assertEquals(2, files.size());
|
||||||
|
|
||||||
|
//////////////////////////////////////
|
||||||
|
// ensure unsupported filters throw //
|
||||||
|
//////////////////////////////////////
|
||||||
|
assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
|
||||||
|
.hasMessageContaining("Unable to query filesystem table by field");
|
||||||
|
assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
|
||||||
|
.hasMessageContaining("Unable to query filename field using operator");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
**
|
**
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
|
@ -23,13 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
|||||||
|
|
||||||
|
|
||||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
|
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
|
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||||
|
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
|
||||||
import org.junit.jupiter.api.Assertions;
|
import org.junit.jupiter.api.Assertions;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
@ -66,4 +72,39 @@ public class S3QueryActionTest extends BaseS3Test
|
|||||||
return queryInput;
|
return queryInput;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
public void testQueryForCardinalityOne() throws QException
|
||||||
|
{
|
||||||
|
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_S3);
|
||||||
|
queryInput.setFilter(new QQueryFilter());
|
||||||
|
|
||||||
|
S3QueryAction s3QueryAction = new S3QueryAction();
|
||||||
|
s3QueryAction.setS3Utils(getS3Utils());
|
||||||
|
|
||||||
|
QueryOutput queryOutput = s3QueryAction.execute(queryInput);
|
||||||
|
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
|
||||||
|
|
||||||
|
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
|
||||||
|
queryOutput = s3QueryAction.execute(queryInput);
|
||||||
|
assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
|
||||||
|
assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////
|
||||||
|
// put a glob on the table - now should only find 2 txt files //
|
||||||
|
////////////////////////////////////////////////////////////////
|
||||||
|
QInstance instance = TestUtils.defineInstance();
|
||||||
|
((S3TableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_S3).getBackendDetails()))
|
||||||
|
.withGlob("*.txt");
|
||||||
|
reInitInstanceInContext(instance);
|
||||||
|
|
||||||
|
queryInput.setFilter(new QQueryFilter());
|
||||||
|
queryOutput = s3QueryAction.execute(queryInput);
|
||||||
|
assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
@ -22,10 +22,10 @@
|
|||||||
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
|
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
|
||||||
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||||
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
@ -42,14 +42,14 @@ public class S3UtilsTest extends BaseS3Test
|
|||||||
**
|
**
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
@Test
|
@Test
|
||||||
public void testListObjectsInBucketAtPath()
|
public void testListObjectsInBucketAtPath() throws QException
|
||||||
{
|
{
|
||||||
S3Utils s3Utils = getS3Utils();
|
S3Utils s3Utils = getS3Utils();
|
||||||
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders");
|
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders");
|
||||||
assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders");
|
assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders");
|
||||||
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders");
|
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders");
|
||||||
assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders");
|
assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders");
|
||||||
assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
|
assertEquals(7, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
|
||||||
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash");
|
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash");
|
||||||
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash");
|
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash");
|
||||||
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes");
|
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes");
|
||||||
@ -60,8 +60,8 @@ public class S3UtilsTest extends BaseS3Test
|
|||||||
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /");
|
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /");
|
||||||
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s");
|
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s");
|
||||||
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string");
|
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string");
|
||||||
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
|
assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
|
||||||
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
|
assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -70,7 +70,7 @@ public class S3UtilsTest extends BaseS3Test
|
|||||||
**
|
**
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
@Test
|
@Test
|
||||||
public void testGetObjectAsInputStream() throws IOException
|
public void testGetObjectAsInputStream() throws Exception
|
||||||
{
|
{
|
||||||
S3Utils s3Utils = getS3Utils();
|
S3Utils s3Utils = getS3Utils();
|
||||||
List<S3ObjectSummary> s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", "");
|
List<S3ObjectSummary> s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", "");
|
||||||
|
Reference in New Issue
Block a user