mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-21 06:28:44 +00:00
Compare commits
1 Commits
snapshot-f
...
snapshot-f
| Author | SHA1 | Date | |
|---|---|---|---|
| e841b37bce |
@ -1178,7 +1178,7 @@ public class BaseAPIActionUtil
|
||||
*******************************************************************************/
|
||||
protected boolean shouldBeRetryableServerErrorException(QHttpResponse qResponse)
|
||||
{
|
||||
if(actionInput instanceof QueryInput || actionInput instanceof GetInput)
|
||||
if(actionInput instanceof QueryInput || actionInput instanceof GetInput || actionInput instanceof UpdateInput)
|
||||
{
|
||||
return (qResponse.getStatusCode() != null && qResponse.getStatusCode() >= 500);
|
||||
}
|
||||
|
||||
@ -36,7 +36,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
@ -69,17 +68,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
|
||||
/*******************************************************************************
|
||||
** List the files for a table - to be implemented in module-specific subclasses.
|
||||
*******************************************************************************/
|
||||
public List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase) throws QException
|
||||
{
|
||||
return (listFiles(table, backendBase, null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses.
|
||||
*******************************************************************************/
|
||||
public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException;
|
||||
public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase);
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the contents of a file - to be implemented in module-specific subclasses.
|
||||
@ -192,7 +181,6 @@ public abstract class AbstractBaseFilesystemAction<FILE>
|
||||
/*******************************************************************************
|
||||
** Generic implementation of the execute method from the QueryInterface
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("checkstyle:Indentation")
|
||||
public QueryOutput executeQuery(QueryInput queryInput) throws QException
|
||||
{
|
||||
preAction(queryInput.getBackend());
|
||||
@ -203,76 +191,51 @@ public abstract class AbstractBaseFilesystemAction<FILE>
|
||||
|
||||
QTableMetaData table = queryInput.getTable();
|
||||
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
|
||||
List<FILE> files = listFiles(table, queryInput.getBackend(), queryInput.getFilter());
|
||||
List<FILE> files = listFiles(table, queryInput.getBackend());
|
||||
|
||||
for(FILE file : files)
|
||||
{
|
||||
LOG.info("Processing file: " + getFullPathForFile(file));
|
||||
|
||||
InputStream inputStream = readFile(file);
|
||||
switch(tableDetails.getCardinality())
|
||||
switch(tableDetails.getRecordFormat())
|
||||
{
|
||||
case MANY:
|
||||
case CSV:
|
||||
{
|
||||
switch(tableDetails.getRecordFormat())
|
||||
String fileContents = IOUtils.toString(readFile(file));
|
||||
fileContents = customizeFileContentsAfterReading(table, fileContents);
|
||||
|
||||
if(queryInput.getRecordPipe() != null)
|
||||
{
|
||||
case CSV:
|
||||
new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record ->
|
||||
{
|
||||
String fileContents = IOUtils.toString(inputStream);
|
||||
fileContents = customizeFileContentsAfterReading(table, fileContents);
|
||||
|
||||
if(queryInput.getRecordPipe() != null)
|
||||
{
|
||||
new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record ->
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Before the records go into the pipe, make sure their backend details are added to them //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
addBackendDetailsToRecord(record, file);
|
||||
}));
|
||||
}
|
||||
else
|
||||
{
|
||||
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
|
||||
addBackendDetailsToRecords(recordsInFile, file);
|
||||
queryOutput.addRecords(recordsInFile);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case JSON:
|
||||
{
|
||||
String fileContents = IOUtils.toString(inputStream);
|
||||
fileContents = customizeFileContentsAfterReading(table, fileContents);
|
||||
|
||||
// todo - pipe support!!
|
||||
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
|
||||
addBackendDetailsToRecords(recordsInFile, file);
|
||||
|
||||
queryOutput.addRecords(recordsInFile);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
{
|
||||
throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat());
|
||||
}
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Before the records go into the pipe, make sure their backend details are added to them //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
addBackendDetailsToRecord(record, file);
|
||||
}));
|
||||
}
|
||||
else
|
||||
{
|
||||
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
|
||||
addBackendDetailsToRecords(recordsInFile, file);
|
||||
queryOutput.addRecords(recordsInFile);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ONE:
|
||||
case JSON:
|
||||
{
|
||||
String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table);
|
||||
String fileContents = IOUtils.toString(readFile(file));
|
||||
fileContents = customizeFileContentsAfterReading(table, fileContents);
|
||||
|
||||
byte[] bytes = inputStream.readAllBytes();
|
||||
QRecord record = new QRecord()
|
||||
.withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase)
|
||||
.withValue(tableDetails.getContentsFieldName(), bytes);
|
||||
queryOutput.addRecord(record);
|
||||
// todo - pipe support!!
|
||||
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
|
||||
addBackendDetailsToRecords(recordsInFile, file);
|
||||
|
||||
queryOutput.addRecords(recordsInFile);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
{
|
||||
throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality());
|
||||
throw new NotImplementedException("Filesystem record format " + tableDetails.getRecordFormat() + " is not yet implemented");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -379,8 +342,8 @@ public abstract class AbstractBaseFilesystemAction<FILE>
|
||||
{
|
||||
for(QRecord record : insertInput.getRecords())
|
||||
{
|
||||
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
|
||||
writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
|
||||
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString("fileName"));
|
||||
writeFile(backend, fullPath, record.getValueByteArray("contents"));
|
||||
record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
|
||||
output.addRecord(record);
|
||||
}
|
||||
|
||||
@ -35,11 +35,6 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
|
||||
private RecordFormat recordFormat;
|
||||
private Cardinality cardinality;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// todo default these to null, and give validation error if not set for a cardinality=ONE table? //
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
private String contentsFieldName = "contents";
|
||||
private String fileNameFieldName = "fileName";
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -180,67 +175,4 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
|
||||
return ((T) this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for contentsFieldName
|
||||
*******************************************************************************/
|
||||
public String getContentsFieldName()
|
||||
{
|
||||
return (this.contentsFieldName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for contentsFieldName
|
||||
*******************************************************************************/
|
||||
public void setContentsFieldName(String contentsFieldName)
|
||||
{
|
||||
this.contentsFieldName = contentsFieldName;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for contentsFieldName
|
||||
*******************************************************************************/
|
||||
public AbstractFilesystemTableBackendDetails withContentsFieldName(String contentsFieldName)
|
||||
{
|
||||
this.contentsFieldName = contentsFieldName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for fileNameFieldName
|
||||
*******************************************************************************/
|
||||
public String getFileNameFieldName()
|
||||
{
|
||||
return (this.fileNameFieldName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for fileNameFieldName
|
||||
*******************************************************************************/
|
||||
public void setFileNameFieldName(String fileNameFieldName)
|
||||
{
|
||||
this.fileNameFieldName = fileNameFieldName;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for fileNameFieldName
|
||||
*******************************************************************************/
|
||||
public AbstractFilesystemTableBackendDetails withFileNameFieldName(String fileNameFieldName)
|
||||
{
|
||||
this.fileNameFieldName = fileNameFieldName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -23,36 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
||||
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileFilter;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.IOCase;
|
||||
import org.apache.commons.io.filefilter.AndFileFilter;
|
||||
import org.apache.commons.io.filefilter.NameFileFilter;
|
||||
import org.apache.commons.io.filefilter.OrFileFilter;
|
||||
import org.apache.commons.io.filefilter.TrueFileFilter;
|
||||
import org.apache.commons.io.filefilter.WildcardFileFilter;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -68,66 +51,12 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
|
||||
** List the files for this table.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
|
||||
public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase)
|
||||
{
|
||||
// todo - needs rewritten to do globbing...
|
||||
String fullPath = getFullBasePath(table, backendBase);
|
||||
File directory = new File(fullPath);
|
||||
File[] files = null;
|
||||
|
||||
AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
|
||||
|
||||
FileFilter fileFilter = TrueFileFilter.INSTANCE;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// if each file is its own record (ONE), then we may need to do filtering of the directory listing based on the input filter //
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(Cardinality.ONE.equals(tableBackendDetails.getCardinality()))
|
||||
{
|
||||
if(filter != null && filter.hasAnyCriteria())
|
||||
{
|
||||
List<FileFilter> fileFilterList = new ArrayList<>();
|
||||
for(QFilterCriteria criteria : filter.getCriteria())
|
||||
{
|
||||
if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
|
||||
{
|
||||
if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
|
||||
{
|
||||
fileFilterList.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(0))));
|
||||
}
|
||||
else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
|
||||
{
|
||||
List<NameFileFilter> nameInFilters = new ArrayList<>();
|
||||
for(int i = 0; i < criteria.getValues().size(); i++)
|
||||
{
|
||||
nameInFilters.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(i))));
|
||||
}
|
||||
fileFilterList.add(new OrFileFilter(nameInFilters));
|
||||
}
|
||||
else
|
||||
{
|
||||
throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
|
||||
}
|
||||
}
|
||||
|
||||
fileFilter = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? new AndFileFilter(fileFilterList) : new OrFileFilter(fileFilterList);
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////
|
||||
// if the table has a glob specified, add it as an AND to the filter built to this point //
|
||||
///////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(StringUtils.hasContent(tableBackendDetails.getGlob()))
|
||||
{
|
||||
WildcardFileFilter globFilenameFilter = new WildcardFileFilter(tableBackendDetails.getGlob(), IOCase.INSENSITIVE);
|
||||
fileFilter = new AndFileFilter(List.of(globFilenameFilter, fileFilter));
|
||||
}
|
||||
|
||||
files = directory.listFiles(fileFilter);
|
||||
File[] files = directory.listFiles();
|
||||
|
||||
if(files == null)
|
||||
{
|
||||
|
||||
@ -59,45 +59,31 @@ public class FilesystemSyncStep implements BackendStep
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// defer to a private method here, so we can add a type-parameter for that method to use //
|
||||
// would think we could do that here, but get compiler error, since this method comes from base class //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
doRun(runBackendStepInput, runBackendStepOutput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private <F> void doRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
|
||||
{
|
||||
QTableMetaData sourceTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
|
||||
QTableMetaData archiveTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
|
||||
QTableMetaData processingTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
|
||||
|
||||
QBackendMetaData sourceBackend = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName());
|
||||
FilesystemBackendModuleInterface<F> sourceModule = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
|
||||
AbstractBaseFilesystemAction<F> sourceActionBase = sourceModule.getActionBase();
|
||||
QBackendMetaData sourceBackend = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName());
|
||||
FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
|
||||
AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase();
|
||||
sourceActionBase.preAction(sourceBackend);
|
||||
Map<String, F> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
|
||||
Map<String, Object> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
|
||||
|
||||
QBackendMetaData archiveBackend = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName());
|
||||
FilesystemBackendModuleInterface<F> archiveModule = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
|
||||
AbstractBaseFilesystemAction<F> archiveActionBase = archiveModule.getActionBase();
|
||||
QBackendMetaData archiveBackend = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName());
|
||||
FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
|
||||
AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase();
|
||||
archiveActionBase.preAction(archiveBackend);
|
||||
Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet();
|
||||
|
||||
QBackendMetaData processingBackend = runBackendStepInput.getInstance().getBackendForTable(processingTable.getName());
|
||||
FilesystemBackendModuleInterface<F> processingModule = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
|
||||
AbstractBaseFilesystemAction<F> processingActionBase = processingModule.getActionBase();
|
||||
QBackendMetaData processingBackend = runBackendStepInput.getInstance().getBackendForTable(processingTable.getName());
|
||||
FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
|
||||
AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase();
|
||||
processingActionBase.preAction(processingBackend);
|
||||
|
||||
Integer maxFilesToSync = runBackendStepInput.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
|
||||
int syncedFileCount = 0;
|
||||
for(Map.Entry<String, F> sourceEntry : sourceFiles.entrySet())
|
||||
for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet())
|
||||
{
|
||||
try
|
||||
{
|
||||
@ -105,22 +91,20 @@ public class FilesystemSyncStep implements BackendStep
|
||||
if(!archiveFiles.contains(sourceFileName))
|
||||
{
|
||||
LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]");
|
||||
try(InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue()))
|
||||
InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue());
|
||||
byte[] bytes = inputStream.readAllBytes();
|
||||
|
||||
String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
|
||||
archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
|
||||
|
||||
String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
|
||||
processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
|
||||
syncedFileCount++;
|
||||
|
||||
if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)
|
||||
{
|
||||
byte[] bytes = inputStream.readAllBytes();
|
||||
|
||||
String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
|
||||
archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
|
||||
|
||||
String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
|
||||
processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
|
||||
syncedFileCount++;
|
||||
|
||||
if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)
|
||||
{
|
||||
LOG.info("Breaking after syncing " + syncedFileCount + " files");
|
||||
break;
|
||||
}
|
||||
LOG.info("Breaking after syncing " + syncedFileCount + " files");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -136,12 +120,12 @@ public class FilesystemSyncStep implements BackendStep
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private <F> Map<String, F> getFileNames(AbstractBaseFilesystemAction<F> actionBase, QTableMetaData table, QBackendMetaData backend) throws QException
|
||||
private Map<String, Object> getFileNames(AbstractBaseFilesystemAction actionBase, QTableMetaData table, QBackendMetaData backend)
|
||||
{
|
||||
List<F> files = actionBase.listFiles(table, backend);
|
||||
Map<String, F> rs = new LinkedHashMap<>();
|
||||
List<Object> files = actionBase.listFiles(table, backend);
|
||||
Map<String, Object> rs = new LinkedHashMap<>();
|
||||
|
||||
for(F file : files)
|
||||
for(Object file : files)
|
||||
{
|
||||
String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table);
|
||||
rs.put(fileName, file);
|
||||
|
||||
@ -30,9 +30,7 @@ import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
@ -128,7 +126,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
|
||||
** List the files for a table.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
|
||||
public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase)
|
||||
{
|
||||
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase);
|
||||
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
|
||||
@ -140,7 +138,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// todo - look at metadata to configure the s3 client here? //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob, filter, tableDetails);
|
||||
return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -37,14 +37,7 @@ import com.amazonaws.services.s3.model.ListObjectsV2Request;
|
||||
import com.amazonaws.services.s3.model.ListObjectsV2Result;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
|
||||
|
||||
@ -68,19 +61,7 @@ public class S3Utils
|
||||
** List the objects in an S3 bucket matching a glob, per:
|
||||
** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
|
||||
*******************************************************************************/
|
||||
public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob) throws QException
|
||||
{
|
||||
return listObjectsInBucketMatchingGlob(bucketName, path, glob, null, null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** List the objects in an S3 bucket matching a glob, per:
|
||||
** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
|
||||
** and also - (possibly) apply a file-name filter (based on the table's details).
|
||||
*******************************************************************************/
|
||||
public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
|
||||
public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob)
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// s3 list requests find nothing if the path starts with a /, so strip away any leading slashes //
|
||||
@ -96,28 +77,6 @@ public class S3Utils
|
||||
prefix = prefix.substring(0, prefix.indexOf('*'));
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// for a file-per-record (ONE) table, we may need to apply the filter to listing. //
|
||||
// but for MANY tables, the filtering would be done on the records after they came out of the files. //
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
boolean useQQueryFilter = false;
|
||||
if(tableDetails != null && Cardinality.ONE.equals(tableDetails.getCardinality()))
|
||||
{
|
||||
useQQueryFilter = true;
|
||||
}
|
||||
|
||||
if(filter != null && useQQueryFilter)
|
||||
{
|
||||
if(filter.getCriteria() != null && filter.getCriteria().size() == 1)
|
||||
{
|
||||
QFilterCriteria criteria = filter.getCriteria().get(0);
|
||||
if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS))
|
||||
{
|
||||
prefix += "/" + criteria.getValues().get(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob. //
|
||||
// That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. //
|
||||
@ -127,7 +86,6 @@ public class S3Utils
|
||||
{
|
||||
glob = "";
|
||||
}
|
||||
|
||||
if(!glob.contains("*"))
|
||||
{
|
||||
if(glob.equals(""))
|
||||
@ -156,7 +114,7 @@ public class S3Utils
|
||||
{
|
||||
listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken());
|
||||
}
|
||||
LOG.info("Listing bucket=" + bucketName + ", path=" + path + ", prefix=" + prefix + ", glob=" + glob);
|
||||
LOG.info("Listing bucket=" + bucketName + ", path=" + path);
|
||||
listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request);
|
||||
|
||||
//////////////////////////////////
|
||||
@ -191,14 +149,6 @@ public class S3Utils
|
||||
continue;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////
|
||||
// if we're a file-per-record table, and we have a filter, compare the key to it //
|
||||
///////////////////////////////////////////////////////////////////////////////////
|
||||
if(!doesObjectKeyMatchFilter(key, filter, tableDetails))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
rs.add(objectSummary);
|
||||
}
|
||||
}
|
||||
@ -209,95 +159,6 @@ public class S3Utils
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private boolean doesObjectKeyMatchFilter(String key, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
|
||||
{
|
||||
if(filter == null || !filter.hasAnyCriteria())
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
|
||||
Path path = Path.of(URI.create("file:///" + key));
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
for(QFilterCriteria criteria : filter.getCriteria())
|
||||
{
|
||||
boolean matches = doesObjectKeyMatchOneCriteria(criteria, tableDetails, path);
|
||||
|
||||
if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// if it's not a match, and it's an AND filter, then the whole thing is false //
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
return (false);
|
||||
}
|
||||
|
||||
if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
|
||||
{
|
||||
////////////////////////////////////////////////////////////
|
||||
// if it's an OR filter, and we've a match, return a true //
|
||||
////////////////////////////////////////////////////////////
|
||||
return (true);
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// if we didn't return above, return now //
|
||||
// for an OR - if we didn't find something true, then return false. //
|
||||
// else, an AND - if we didn't find a false, we can return true. //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static boolean doesObjectKeyMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException
|
||||
{
|
||||
if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
|
||||
{
|
||||
if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
|
||||
{
|
||||
return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path));
|
||||
}
|
||||
else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
|
||||
{
|
||||
boolean anyMatch = false;
|
||||
for(int i = 0; i < criteria.getValues().size(); i++)
|
||||
{
|
||||
if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path))
|
||||
{
|
||||
anyMatch = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return (anyMatch);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Get the contents (as an InputStream) for an object in s3
|
||||
*******************************************************************************/
|
||||
@ -384,5 +245,4 @@ public class S3Utils
|
||||
|
||||
return amazonS3;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -25,11 +25,6 @@ package com.kingsrook.qqq.backend.module.filesystem.local;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||
@ -41,8 +36,6 @@ import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -54,9 +47,6 @@ public class FilesystemBackendModuleTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
public void beforeEach() throws IOException
|
||||
{
|
||||
@ -65,9 +55,6 @@ public class FilesystemBackendModuleTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
public void afterEach() throws Exception
|
||||
{
|
||||
@ -76,78 +63,6 @@ public class FilesystemBackendModuleTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testListFiles() throws QException
|
||||
{
|
||||
QInstance qInstance = TestUtils.defineInstance();
|
||||
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
|
||||
|
||||
AbstractFilesystemAction abstractFilesystemAction = new AbstractFilesystemAction();
|
||||
QBackendMetaData backend = qInstance.getBackendForTable(table.getName());
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
// with no filter given, all (3) files should come back //
|
||||
//////////////////////////////////////////////////////////
|
||||
List<File> files = abstractFilesystemAction.listFiles(table, backend);
|
||||
assertEquals(3, files.size());
|
||||
|
||||
/////////////////////////////////////////
|
||||
// filter for a file name that's found //
|
||||
/////////////////////////////////////////
|
||||
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
|
||||
assertEquals(1, files.size());
|
||||
assertEquals("BLOB-2.txt", files.get(0).getName());
|
||||
|
||||
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
|
||||
assertEquals(1, files.size());
|
||||
assertEquals("BLOB-1.txt", files.get(0).getName());
|
||||
|
||||
///////////////////////////////////
|
||||
// filter for 2 names that exist //
|
||||
///////////////////////////////////
|
||||
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
|
||||
assertEquals(2, files.size());
|
||||
|
||||
/////////////////////////////////////////////
|
||||
// filter for a file name that isn't found //
|
||||
/////////////////////////////////////////////
|
||||
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
|
||||
assertEquals(0, files.size());
|
||||
|
||||
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
|
||||
assertEquals(1, files.size());
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 2 criteria, and'ed, and can't match, so find 0 //
|
||||
////////////////////////////////////////////////////
|
||||
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
|
||||
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
|
||||
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
|
||||
assertEquals(0, files.size());
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// 2 criteria, or'ed, and both match, so find 2 //
|
||||
//////////////////////////////////////////////////
|
||||
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
|
||||
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
|
||||
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
|
||||
.withBooleanOperator(QQueryFilter.BooleanOperator.OR));
|
||||
assertEquals(2, files.size());
|
||||
|
||||
//////////////////////////////////////
|
||||
// ensure unsupported filters throw //
|
||||
//////////////////////////////////////
|
||||
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
|
||||
.hasMessageContaining("Unable to query filesystem table by field");
|
||||
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
|
||||
.hasMessageContaining("Unable to query filename field using operator");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
|
||||
@ -89,7 +89,6 @@ public class FilesystemActionTest extends BaseTest
|
||||
|
||||
writePersonJSONFiles(baseDirectory);
|
||||
writePersonCSVFiles(baseDirectory);
|
||||
writeBlobFiles(baseDirectory);
|
||||
}
|
||||
|
||||
|
||||
@ -159,37 +158,6 @@ public class FilesystemActionTest extends BaseTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Write some data files into the directory for the filesystem module.
|
||||
*******************************************************************************/
|
||||
private void writeBlobFiles(File baseDirectory) throws IOException
|
||||
{
|
||||
String fullPath = baseDirectory.getAbsolutePath();
|
||||
if(TestUtils.defineLocalFilesystemBlobTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
|
||||
{
|
||||
if(StringUtils.hasContent(details.getBasePath()))
|
||||
{
|
||||
fullPath += File.separatorChar + details.getBasePath();
|
||||
}
|
||||
}
|
||||
fullPath += File.separatorChar;
|
||||
|
||||
String data1 = """
|
||||
Hello, Blob
|
||||
""";
|
||||
FileUtils.writeStringToFile(new File(fullPath + "BLOB-1.txt"), data1);
|
||||
|
||||
String data2 = """
|
||||
Hi Bob""";
|
||||
FileUtils.writeStringToFile(new File(fullPath + "BLOB-2.txt"), data2);
|
||||
|
||||
String data3 = """
|
||||
# Hi MD...""";
|
||||
FileUtils.writeStringToFile(new File(fullPath + "BLOB-3.md"), data3);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
|
||||
@ -23,9 +23,6 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
@ -35,10 +32,8 @@ import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractPostReadFileCustomizer;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.actions.FilesystemTableCustomizers;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -56,8 +51,8 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
|
||||
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
|
||||
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
|
||||
assertTrue(queryOutput.getRecords().stream()
|
||||
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
|
||||
Assertions.assertTrue(queryOutput.getRecords().stream()
|
||||
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
|
||||
"All records should have a full-path in their backend details, matching the test folder name");
|
||||
}
|
||||
@ -79,48 +74,14 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
|
||||
|
||||
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
|
||||
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
|
||||
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
|
||||
assertTrue(
|
||||
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
|
||||
Assertions.assertTrue(
|
||||
queryOutput.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")),
|
||||
"All records should have their email addresses up-shifted.");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testQueryForCardinalityOne() throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
|
||||
queryInput.setFilter(new QQueryFilter());
|
||||
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
|
||||
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
|
||||
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
|
||||
queryOutput = new FilesystemQueryAction().execute(queryInput);
|
||||
assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
|
||||
assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// put a glob on the table - now should only find 2 txt files //
|
||||
////////////////////////////////////////////////////////////////
|
||||
QInstance instance = TestUtils.defineInstance();
|
||||
((FilesystemTableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).getBackendDetails()))
|
||||
.withGlob("*.txt");
|
||||
reInitInstanceInContext(instance);
|
||||
|
||||
queryInput.setFilter(new QQueryFilter());
|
||||
queryOutput = new FilesystemQueryAction().execute(queryInput);
|
||||
assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static class ValueUpshifter extends AbstractPostReadFileCustomizer
|
||||
{
|
||||
@Override
|
||||
|
||||
@ -26,7 +26,7 @@ import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
@ -187,7 +187,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QException
|
||||
private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException
|
||||
{
|
||||
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
|
||||
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
|
||||
@ -207,7 +207,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QException
|
||||
private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException
|
||||
{
|
||||
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
|
||||
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
|
||||
|
||||
@ -62,9 +62,6 @@ public class BaseS3Test extends BaseTest
|
||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2());
|
||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test");
|
||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
|
||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-1.txt", "Hello, Blob");
|
||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-2.txt", "Hi, Bob");
|
||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-3.md", "# Hi, MD");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -25,11 +25,6 @@ package com.kingsrook.qqq.backend.module.filesystem.s3;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||
@ -37,9 +32,6 @@ import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemExceptio
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -51,83 +43,6 @@ public class S3BackendModuleTest extends BaseS3Test
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testListFiles() throws QException
|
||||
{
|
||||
QInstance qInstance = TestUtils.defineInstance();
|
||||
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_S3);
|
||||
QBackendMetaData backend = qInstance.getBackendForTable(table.getName());
|
||||
|
||||
//////////////////////////////////////////////////////
|
||||
// set up the backend module (e.g., for localstack) //
|
||||
//////////////////////////////////////////////////////
|
||||
S3BackendModule s3BackendModule = new S3BackendModule();
|
||||
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
|
||||
actionBase.setS3Utils(getS3Utils());
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
// with no filter given, all (3) files should come back //
|
||||
//////////////////////////////////////////////////////////
|
||||
List<S3ObjectSummary> files = actionBase.listFiles(table, backend);
|
||||
assertEquals(3, files.size());
|
||||
|
||||
/////////////////////////////////////////
|
||||
// filter for a file name that's found //
|
||||
/////////////////////////////////////////
|
||||
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
|
||||
assertEquals(1, files.size());
|
||||
assertThat(files.get(0).getKey()).contains("BLOB-2.txt");
|
||||
|
||||
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
|
||||
assertEquals(1, files.size());
|
||||
assertThat(files.get(0).getKey()).contains("BLOB-1.txt");
|
||||
|
||||
///////////////////////////////////
|
||||
// filter for 2 names that exist //
|
||||
///////////////////////////////////
|
||||
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
|
||||
assertEquals(2, files.size());
|
||||
|
||||
/////////////////////////////////////////////
|
||||
// filter for a file name that isn't found //
|
||||
/////////////////////////////////////////////
|
||||
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
|
||||
assertEquals(0, files.size());
|
||||
|
||||
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
|
||||
assertEquals(1, files.size());
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 2 criteria, and'ed, and can't match, so find 0 //
|
||||
////////////////////////////////////////////////////
|
||||
files = actionBase.listFiles(table, backend, new QQueryFilter(
|
||||
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
|
||||
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
|
||||
assertEquals(0, files.size());
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// 2 criteria, or'ed, and both match, so find 2 //
|
||||
//////////////////////////////////////////////////
|
||||
files = actionBase.listFiles(table, backend, new QQueryFilter(
|
||||
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
|
||||
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
|
||||
.withBooleanOperator(QQueryFilter.BooleanOperator.OR));
|
||||
assertEquals(2, files.size());
|
||||
|
||||
//////////////////////////////////////
|
||||
// ensure unsupported filters throw //
|
||||
//////////////////////////////////////
|
||||
assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
|
||||
.hasMessageContaining("Unable to query filesystem table by field");
|
||||
assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
|
||||
.hasMessageContaining("Unable to query filename field using operator");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
|
||||
@ -23,19 +23,13 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -72,39 +66,4 @@ public class S3QueryActionTest extends BaseS3Test
|
||||
return queryInput;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testQueryForCardinalityOne() throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_S3);
|
||||
queryInput.setFilter(new QQueryFilter());
|
||||
|
||||
S3QueryAction s3QueryAction = new S3QueryAction();
|
||||
s3QueryAction.setS3Utils(getS3Utils());
|
||||
|
||||
QueryOutput queryOutput = s3QueryAction.execute(queryInput);
|
||||
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
|
||||
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
|
||||
queryOutput = s3QueryAction.execute(queryInput);
|
||||
assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
|
||||
assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// put a glob on the table - now should only find 2 txt files //
|
||||
////////////////////////////////////////////////////////////////
|
||||
QInstance instance = TestUtils.defineInstance();
|
||||
((S3TableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_S3).getBackendDetails()))
|
||||
.withGlob("*.txt");
|
||||
reInitInstanceInContext(instance);
|
||||
|
||||
queryInput.setFilter(new QQueryFilter());
|
||||
queryOutput = s3QueryAction.execute(queryInput);
|
||||
assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
|
||||
}
|
||||
|
||||
}
|
||||
@ -22,10 +22,10 @@
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.List;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@ -42,14 +42,14 @@ public class S3UtilsTest extends BaseS3Test
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testListObjectsInBucketAtPath() throws QException
|
||||
public void testListObjectsInBucketAtPath()
|
||||
{
|
||||
S3Utils s3Utils = getS3Utils();
|
||||
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders");
|
||||
assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders");
|
||||
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders");
|
||||
assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders");
|
||||
assertEquals(7, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
|
||||
assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
|
||||
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash");
|
||||
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash");
|
||||
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes");
|
||||
@ -60,8 +60,8 @@ public class S3UtilsTest extends BaseS3Test
|
||||
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /");
|
||||
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s");
|
||||
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string");
|
||||
assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
|
||||
assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
|
||||
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
|
||||
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
|
||||
}
|
||||
|
||||
|
||||
@ -70,7 +70,7 @@ public class S3UtilsTest extends BaseS3Test
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testGetObjectAsInputStream() throws Exception
|
||||
public void testGetObjectAsInputStream() throws IOException
|
||||
{
|
||||
S3Utils s3Utils = getS3Utils();
|
||||
List<S3ObjectSummary> s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", "");
|
||||
|
||||
Reference in New Issue
Block a user