QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #55 from Kingsrook/feature/CE-773-cartonization-playground

Browse Source 
Feature/ce 773 cartonization playground
This commit is contained in:
darin.kelkhoff
2023-12-28 18:45:30 -06:00
committed by GitHub
parent 78d9ec87a2 6e1ea5c8f1
commit 959f8c8041
21 changed files with 1057 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java
View File

@ -132,6 +132,15 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
//////////////////////////////////// ////////////////////////////////////
InsertOutput insertOutput = insertInterface.execute(insertInput); InsertOutput insertOutput = insertInterface.execute(insertInput);
if(insertOutput.getRecords() == null)
{
////////////////////////////////////////////////////////////////////////////////////
// in case the module failed to set record in the output, put an empty list there //
// to avoid so many downstream NPE's //
////////////////////////////////////////////////////////////////////////////////////
insertOutput.setRecords(new ArrayList<>());
}
////////////////////////////// //////////////////////////////
// log if there were errors // // log if there were errors //
////////////////////////////// //////////////////////////////

9
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java
View File

@ -137,6 +137,15 @@ public class UpdateAction
//////////////////////////////////// ////////////////////////////////////
UpdateOutput updateOutput = updateInterface.execute(updateInput); UpdateOutput updateOutput = updateInterface.execute(updateInput);
if(updateOutput.getRecords() == null)
{
////////////////////////////////////////////////////////////////////////////////////
// in case the module failed to set record in the output, put an empty list there //
// to avoid so many downstream NPE's //
////////////////////////////////////////////////////////////////////////////////////
updateOutput.setRecords(new ArrayList<>());
}
////////////////////////////// //////////////////////////////
// log if there were errors // // log if there were errors //
////////////////////////////// //////////////////////////////

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
View File

@ -480,6 +480,11 @@ public class QInstanceValidator
validateTableCustomizer(tableName, entry.getKey(), entry.getValue()); validateTableCustomizer(tableName, entry.getKey(), entry.getValue());
} }
if(table.getBackendDetails() != null)
{
table.getBackendDetails().validate(qInstance, table, this);
}
validateTableAutomationDetails(qInstance, table); validateTableAutomationDetails(qInstance, table);
validateTableUniqueKeys(table); validateTableUniqueKeys(table);
validateAssociatedScripts(table); validateAssociatedScripts(table);

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableBackendDetails.java
View File

@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.serialization.QTableBackendDetailsDeserializer; import com.kingsrook.qqq.backend.core.model.metadata.serialization.QTableBackendDetailsDeserializer;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@ -100,4 +103,16 @@ public abstract class QTableBackendDetails
return (this); return (this);
} }
/*******************************************************************************
**
*******************************************************************************/
public void validate(QInstance qInstance, QTableMetaData table, QInstanceValidator qInstanceValidator)
{
////////////////////////
// noop in base class //
////////////////////////
}
} }

124
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.filesystem.base.actions;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
@ -36,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -51,6 +53,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinali
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.NotImplementedException; import org.apache.commons.lang.NotImplementedException;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/******************************************************************************* /*******************************************************************************
@ -68,7 +71,17 @@ public abstract class AbstractBaseFilesystemAction<FILE>
/******************************************************************************* /*******************************************************************************
** List the files for a table - to be implemented in module-specific subclasses. ** List the files for a table - to be implemented in module-specific subclasses.
*******************************************************************************/ *******************************************************************************/
public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase); public List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase) throws QException
{
return (listFiles(table, backendBase, null));
}
/*******************************************************************************
** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses.
*******************************************************************************/
public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException;
/******************************************************************************* /*******************************************************************************
** Read the contents of a file - to be implemented in module-specific subclasses. ** Read the contents of a file - to be implemented in module-specific subclasses.
@ -181,6 +194,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
/******************************************************************************* /*******************************************************************************
** Generic implementation of the execute method from the QueryInterface ** Generic implementation of the execute method from the QueryInterface
*******************************************************************************/ *******************************************************************************/
@SuppressWarnings("checkstyle:Indentation")
public QueryOutput executeQuery(QueryInput queryInput) throws QException public QueryOutput executeQuery(QueryInput queryInput) throws QException
{ {
preAction(queryInput.getBackend()); preAction(queryInput.getBackend());
@ -191,51 +205,97 @@ public abstract class AbstractBaseFilesystemAction<FILE>
QTableMetaData table = queryInput.getTable(); QTableMetaData table = queryInput.getTable();
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
List<FILE> files = listFiles(table, queryInput.getBackend()); List<FILE> files = listFiles(table, queryInput.getBackend(), queryInput.getFilter());
int recordCount = 0;
FILE_LOOP:
for(FILE file : files) for(FILE file : files)
{ {
LOG.info("Processing file: " + getFullPathForFile(file)); InputStream inputStream = readFile(file);
switch(tableDetails.getRecordFormat()) switch(tableDetails.getCardinality())
{ {
case CSV: case MANY:
{ {
String fileContents = IOUtils.toString(readFile(file)); LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file)));
fileContents = customizeFileContentsAfterReading(table, fileContents); switch(tableDetails.getRecordFormat())
if(queryInput.getRecordPipe() != null)
{ {
new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> case CSV:
{ {
//////////////////////////////////////////////////////////////////////////////////////////// String fileContents = IOUtils.toString(inputStream);
// Before the records go into the pipe, make sure their backend details are added to them // fileContents = customizeFileContentsAfterReading(table, fileContents);
////////////////////////////////////////////////////////////////////////////////////////////
addBackendDetailsToRecord(record, file); if(queryInput.getRecordPipe() != null)
})); {
} new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record ->
else {
{ ////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); // Before the records go into the pipe, make sure their backend details are added to them //
addBackendDetailsToRecords(recordsInFile, file); ////////////////////////////////////////////////////////////////////////////////////////////
queryOutput.addRecords(recordsInFile); addBackendDetailsToRecord(record, file);
}));
}
else
{
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
queryOutput.addRecords(recordsInFile);
}
break;
}
case JSON:
{
String fileContents = IOUtils.toString(inputStream);
fileContents = customizeFileContentsAfterReading(table, fileContents);
// todo - pipe support!!
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
queryOutput.addRecords(recordsInFile);
break;
}
default:
{
throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat());
}
} }
break; break;
} }
case JSON: case ONE:
{ {
String fileContents = IOUtils.toString(readFile(file)); ////////////////////////////////////////////////////////////////////////////////
fileContents = customizeFileContentsAfterReading(table, fileContents); // for one-record tables, put the entire file's contents into a single record //
////////////////////////////////////////////////////////////////////////////////
String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table);
byte[] bytes = inputStream.readAllBytes();
// todo - pipe support!! QRecord record = new QRecord()
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); .withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase)
addBackendDetailsToRecords(recordsInFile, file); .withValue(tableDetails.getContentsFieldName(), bytes);
queryOutput.addRecord(record);
///////////////////////////////////////////////////////////////////////////////////////////////////////////
// keep our own count - in case the query output is using a pipe (e.g., so we can't just call a .size()) //
///////////////////////////////////////////////////////////////////////////////////////////////////////////
recordCount++;
////////////////////////////////////////////////////////////////////////////
// break out of the file loop if we have hit the limit (if one was given) //
////////////////////////////////////////////////////////////////////////////
if(queryInput.getFilter() != null && queryInput.getFilter().getLimit() != null)
{
if(recordCount >= queryInput.getFilter().getLimit())
{
break FILE_LOOP;
}
}
queryOutput.addRecords(recordsInFile);
break; break;
} }
default: default:
{ {
throw new NotImplementedException("Filesystem record format " + tableDetails.getRecordFormat() + " is not yet implemented"); throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality());
} }
} }
} }
@ -337,13 +397,15 @@ public abstract class AbstractBaseFilesystemAction<FILE>
QTableMetaData table = insertInput.getTable(); QTableMetaData table = insertInput.getTable();
QBackendMetaData backend = insertInput.getBackend(); QBackendMetaData backend = insertInput.getBackend();
output.setRecords(new ArrayList<>());
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
if(tableDetails.getCardinality().equals(Cardinality.ONE)) if(tableDetails.getCardinality().equals(Cardinality.ONE))
{ {
for(QRecord record : insertInput.getRecords()) for(QRecord record : insertInput.getRecords())
{ {
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString("fileName")); String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
writeFile(backend, fullPath, record.getValueByteArray("contents")); writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath); record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
output.addRecord(record); output.addRecord(record);
} }

106
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java
View File

@ -22,7 +22,11 @@
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/******************************************************************************* /*******************************************************************************
@ -35,6 +39,9 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
private RecordFormat recordFormat; private RecordFormat recordFormat;
private Cardinality cardinality; private Cardinality cardinality;
private String contentsFieldName;
private String fileNameFieldName;
/******************************************************************************* /*******************************************************************************
@ -175,4 +182,103 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
return ((T) this); return ((T) this);
} }
/*******************************************************************************
** Getter for contentsFieldName
*******************************************************************************/
public String getContentsFieldName()
{
return (this.contentsFieldName);
}
/*******************************************************************************
** Setter for contentsFieldName
*******************************************************************************/
public void setContentsFieldName(String contentsFieldName)
{
this.contentsFieldName = contentsFieldName;
}
/*******************************************************************************
** Fluent setter for contentsFieldName
*******************************************************************************/
public AbstractFilesystemTableBackendDetails withContentsFieldName(String contentsFieldName)
{
this.contentsFieldName = contentsFieldName;
return (this);
}
/*******************************************************************************
** Getter for fileNameFieldName
*******************************************************************************/
public String getFileNameFieldName()
{
return (this.fileNameFieldName);
}
/*******************************************************************************
** Setter for fileNameFieldName
*******************************************************************************/
public void setFileNameFieldName(String fileNameFieldName)
{
this.fileNameFieldName = fileNameFieldName;
}
/*******************************************************************************
** Fluent setter for fileNameFieldName
*******************************************************************************/
public AbstractFilesystemTableBackendDetails withFileNameFieldName(String fileNameFieldName)
{
this.fileNameFieldName = fileNameFieldName;
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void validate(QInstance qInstance, QTableMetaData table, QInstanceValidator qInstanceValidator)
{
super.validate(qInstance, table, qInstanceValidator);
String prefix = "Table " + (table == null ? "null" : table.getName()) + " backend details - ";
if(qInstanceValidator.assertCondition(cardinality != null, prefix + "missing cardinality"))
{
if(cardinality.equals(Cardinality.ONE))
{
if(qInstanceValidator.assertCondition(StringUtils.hasContent(contentsFieldName), prefix + "missing contentsFieldName, which is required for Cardinality ONE"))
{
qInstanceValidator.assertCondition(table != null && table.getFields().containsKey(contentsFieldName), prefix + "contentsFieldName [" + contentsFieldName + "] is not a field on this table.");
}
if(qInstanceValidator.assertCondition(StringUtils.hasContent(fileNameFieldName), prefix + "missing fileNameFieldName, which is required for Cardinality ONE"))
{
qInstanceValidator.assertCondition(table != null && table.getFields().containsKey(fileNameFieldName), prefix + "fileNameFieldName [" + fileNameFieldName + "] is not a field on this table.");
}
qInstanceValidator.assertCondition(recordFormat == null, prefix + "has a recordFormat, which is not allowed for Cardinality ONE");
}
if(cardinality.equals(Cardinality.MANY))
{
qInstanceValidator.assertCondition(!StringUtils.hasContent(contentsFieldName), prefix + "has a contentsFieldName, which is not allowed for Cardinality MANY");
qInstanceValidator.assertCondition(!StringUtils.hasContent(fileNameFieldName), prefix + "has a fileNameFieldName, which is not allowed for Cardinality MANY");
qInstanceValidator.assertCondition(recordFormat != null, prefix + "missing recordFormat, which is required for Cardinality MANY");
}
}
}
} }

85
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java
View File

@ -23,19 +23,36 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.io.File; import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOCase;
import org.apache.commons.io.filefilter.AndFileFilter;
import org.apache.commons.io.filefilter.NameFileFilter;
import org.apache.commons.io.filefilter.OrFileFilter;
import org.apache.commons.io.filefilter.TrueFileFilter;
import org.apache.commons.io.filefilter.WildcardFileFilter;
/******************************************************************************* /*******************************************************************************
@ -51,12 +68,74 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
** List the files for this table. ** List the files for this table.
*******************************************************************************/ *******************************************************************************/
@Override @Override
public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase) public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
{ {
// todo - needs rewritten to do globbing...
String fullPath = getFullBasePath(table, backendBase); String fullPath = getFullBasePath(table, backendBase);
File directory = new File(fullPath); File directory = new File(fullPath);
File[] files = directory.listFiles(); File[] files = null;
AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
FileFilter fileFilter = TrueFileFilter.INSTANCE;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if each file is its own record (ONE), then we may need to do filtering of the directory listing based on the input filter //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(Cardinality.ONE.equals(tableBackendDetails.getCardinality()))
{
if(filter != null && filter.hasAnyCriteria())
{
if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
{
///////////////////////////////////////////////////////////////////////
// todo - well, we could - just build up a tree of and's and or's... //
///////////////////////////////////////////////////////////////////////
throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time."));
}
List<FileFilter> fileFilterList = new ArrayList<>();
for(QFilterCriteria criteria : filter.getCriteria())
{
if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
{
if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
{
fileFilterList.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(0))));
}
else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
{
List<NameFileFilter> nameInFilters = new ArrayList<>();
for(int i = 0; i < criteria.getValues().size(); i++)
{
nameInFilters.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(i))));
}
fileFilterList.add(new OrFileFilter(nameInFilters));
}
else
{
throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
}
}
else
{
throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
}
}
fileFilter = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? new AndFileFilter(fileFilterList) : new OrFileFilter(fileFilterList);
}
}
///////////////////////////////////////////////////////////////////////////////////////////
// if the table has a glob specified, add it as an AND to the filter built to this point //
///////////////////////////////////////////////////////////////////////////////////////////
if(StringUtils.hasContent(tableBackendDetails.getGlob()))
{
WildcardFileFilter globFilenameFilter = new WildcardFileFilter(tableBackendDetails.getGlob(), IOCase.INSENSITIVE);
fileFilter = new AndFileFilter(List.of(globFilenameFilter, fileFilter));
}
files = directory.listFiles(fileFilter);
if(files == null) if(files == null)
{ {

72
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java
View File

@ -59,31 +59,45 @@ public class FilesystemSyncStep implements BackendStep
*******************************************************************************/ *******************************************************************************/
@Override @Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
////////////////////////////////////////////////////////////////////////////////////////////////////////
// defer to a private method here, so we can add a type-parameter for that method to use //
// would think we could do that here, but get compiler error, since this method comes from base class //
////////////////////////////////////////////////////////////////////////////////////////////////////////
doRun(runBackendStepInput, runBackendStepOutput);
}
/*******************************************************************************
**
*******************************************************************************/
private <F> void doRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{ {
QTableMetaData sourceTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE)); QTableMetaData sourceTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
QTableMetaData archiveTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE)); QTableMetaData archiveTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
QTableMetaData processingTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE)); QTableMetaData processingTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
QBackendMetaData sourceBackend = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName()); QBackendMetaData sourceBackend = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName());
FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend); FilesystemBackendModuleInterface<F> sourceModule = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase(); AbstractBaseFilesystemAction<F> sourceActionBase = sourceModule.getActionBase();
sourceActionBase.preAction(sourceBackend); sourceActionBase.preAction(sourceBackend);
Map<String, Object> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend); Map<String, F> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
QBackendMetaData archiveBackend = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName()); QBackendMetaData archiveBackend = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName());
FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend); FilesystemBackendModuleInterface<F> archiveModule = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase(); AbstractBaseFilesystemAction<F> archiveActionBase = archiveModule.getActionBase();
archiveActionBase.preAction(archiveBackend); archiveActionBase.preAction(archiveBackend);
Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet(); Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet();
QBackendMetaData processingBackend = runBackendStepInput.getInstance().getBackendForTable(processingTable.getName()); QBackendMetaData processingBackend = runBackendStepInput.getInstance().getBackendForTable(processingTable.getName());
FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend); FilesystemBackendModuleInterface<F> processingModule = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase(); AbstractBaseFilesystemAction<F> processingActionBase = processingModule.getActionBase();
processingActionBase.preAction(processingBackend); processingActionBase.preAction(processingBackend);
Integer maxFilesToSync = runBackendStepInput.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE); Integer maxFilesToSync = runBackendStepInput.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
int syncedFileCount = 0; int syncedFileCount = 0;
for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet()) for(Map.Entry<String, F> sourceEntry : sourceFiles.entrySet())
{ {
try try
{ {
@ -91,20 +105,22 @@ public class FilesystemSyncStep implements BackendStep
if(!archiveFiles.contains(sourceFileName)) if(!archiveFiles.contains(sourceFileName))
{ {
LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]"); LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]");
InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue()); try(InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue()))
byte[] bytes = inputStream.readAllBytes();
String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
syncedFileCount++;
if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)
{ {
LOG.info("Breaking after syncing " + syncedFileCount + " files"); byte[] bytes = inputStream.readAllBytes();
break;
String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
syncedFileCount++;
if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)
{
LOG.info("Breaking after syncing " + syncedFileCount + " files");
break;
}
} }
} }
} }
@ -120,12 +136,12 @@ public class FilesystemSyncStep implements BackendStep
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/
private Map<String, Object> getFileNames(AbstractBaseFilesystemAction actionBase, QTableMetaData table, QBackendMetaData backend) private <F> Map<String, F> getFileNames(AbstractBaseFilesystemAction<F> actionBase, QTableMetaData table, QBackendMetaData backend) throws QException
{ {
List<Object> files = actionBase.listFiles(table, backend); List<F> files = actionBase.listFiles(table, backend);
Map<String, Object> rs = new LinkedHashMap<>(); Map<String, F> rs = new LinkedHashMap<>();
for(Object file : files) for(F file : files)
{ {
String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table); String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table);
rs.put(fileName, file); rs.put(fileName, file);

6
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java
View File

@ -30,7 +30,9 @@ import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@ -126,7 +128,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
** List the files for a table. ** List the files for a table.
*******************************************************************************/ *******************************************************************************/
@Override @Override
public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase) public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
{ {
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase); S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase);
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
@ -138,7 +140,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
// todo - look at metadata to configure the s3 client here? // // todo - look at metadata to configure the s3 client here? //
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob); return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob, filter, tableDetails);
} }

168
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java
View File

@ -37,7 +37,14 @@ import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result; import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
@ -61,7 +68,19 @@ public class S3Utils
** List the objects in an S3 bucket matching a glob, per: ** List the objects in an S3 bucket matching a glob, per:
** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String) ** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
*******************************************************************************/ *******************************************************************************/
public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob) public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob) throws QException
{
return listObjectsInBucketMatchingGlob(bucketName, path, glob, null, null);
}
/*******************************************************************************
** List the objects in an S3 bucket matching a glob, per:
** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
** and also - (possibly) apply a file-name filter (based on the table's details).
*******************************************************************************/
public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
{ {
////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////
// s3 list requests find nothing if the path starts with a /, so strip away any leading slashes // // s3 list requests find nothing if the path starts with a /, so strip away any leading slashes //
@ -77,6 +96,32 @@ public class S3Utils
prefix = prefix.substring(0, prefix.indexOf('*')); prefix = prefix.substring(0, prefix.indexOf('*'));
} }
///////////////////////////////////////////////////////////////////////////////////////////////////////
// for a file-per-record (ONE) table, we may need to apply the filter to listing. //
// but for MANY tables, the filtering would be done on the records after they came out of the files. //
///////////////////////////////////////////////////////////////////////////////////////////////////////
boolean useQQueryFilter = false;
if(tableDetails != null && Cardinality.ONE.equals(tableDetails.getCardinality()))
{
useQQueryFilter = true;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if there's a filter for single file, make that file name the "prefix" that we send to s3, so we just get back that 1 file. //
// as this will be a common case. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(filter != null && useQQueryFilter)
{
if(filter.getCriteria() != null && filter.getCriteria().size() == 1)
{
QFilterCriteria criteria = filter.getCriteria().get(0);
if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS))
{
prefix += "/" + criteria.getValues().get(0);
}
}
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob. // // mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob. //
// That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. // // That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. //
@ -86,6 +131,7 @@ public class S3Utils
{ {
glob = ""; glob = "";
} }
if(!glob.contains("*")) if(!glob.contains("*"))
{ {
if(glob.equals("")) if(glob.equals(""))
@ -114,7 +160,7 @@ public class S3Utils
{ {
listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken()); listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken());
} }
LOG.info("Listing bucket=" + bucketName + ", path=" + path); LOG.info("Listing bucket=" + bucketName + ", path=" + path + ", prefix=" + prefix + ", glob=" + glob);
listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request); listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request);
////////////////////////////////// //////////////////////////////////
@ -149,7 +195,27 @@ public class S3Utils
continue; continue;
} }
///////////////////////////////////////////////////////////////////////////////////
// if we're a file-per-record table, and we have a filter, compare the key to it //
///////////////////////////////////////////////////////////////////////////////////
if(!doesObjectKeyMatchFilter(key, filter, tableDetails))
{
continue;
}
rs.add(objectSummary); rs.add(objectSummary);
/////////////////////////////////////////////////////////////////
// if we have a limit, and we've hit it, break out of the loop //
/////////////////////////////////////////////////////////////////
if(filter != null && useQQueryFilter && filter.getLimit() != null)
{
if(rs.size() >= filter.getLimit())
{
break;
}
}
} }
} }
while(listObjectsV2Result.isTruncated()); while(listObjectsV2Result.isTruncated());
@ -159,6 +225,103 @@ public class S3Utils
/*******************************************************************************
**
*******************************************************************************/
private boolean doesObjectKeyMatchFilter(String key, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
{
if(filter == null || !filter.hasAnyCriteria())
{
return (true);
}
if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
{
///////////////////////////////
// todo - well, we could ... //
///////////////////////////////
throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time."));
}
Path path = Path.of(URI.create("file:///" + key));
////////////////////////////////////////////////////////////////////////////////////////////////////
// foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches //
////////////////////////////////////////////////////////////////////////////////////////////////////
for(QFilterCriteria criteria : filter.getCriteria())
{
boolean matches = doesObjectKeyMatchOneCriteria(criteria, tableDetails, path);
if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
{
////////////////////////////////////////////////////////////////////////////////
// if it's not a match, and it's an AND filter, then the whole thing is false //
////////////////////////////////////////////////////////////////////////////////
return (false);
}
if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
{
////////////////////////////////////////////////////////////
// if it's an OR filter, and we've a match, return a true //
////////////////////////////////////////////////////////////
return (true);
}
}
//////////////////////////////////////////////////////////////////////
// if we didn't return above, return now //
// for an OR - if we didn't find something true, then return false. //
// else, an AND - if we didn't find a false, we can return true. //
//////////////////////////////////////////////////////////////////////
if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
{
return (false);
}
return (true);
}
/*******************************************************************************
**
*******************************************************************************/
private static boolean doesObjectKeyMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException
{
if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
{
if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
{
return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path));
}
else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
{
boolean anyMatch = false;
for(int i = 0; i < criteria.getValues().size(); i++)
{
if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path))
{
anyMatch = true;
break;
}
}
return (anyMatch);
}
else
{
throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
}
}
else
{
throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
}
}
/******************************************************************************* /*******************************************************************************
** Get the contents (as an InputStream) for an object in s3 ** Get the contents (as an InputStream) for an object in s3
*******************************************************************************/ *******************************************************************************/
@ -245,4 +408,5 @@ public class S3Utils
return amazonS3; return amazonS3;
} }
} }

7
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -26,7 +26,6 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -142,8 +141,6 @@ public class TestUtils
qInstance.addTable(defineMockPersonTable()); qInstance.addTable(defineMockPersonTable());
qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess()); qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
new QInstanceValidator().validate(qInstance);
return (qInstance); return (qInstance);
} }
@ -249,6 +246,8 @@ public class TestUtils
.withBackendDetails(new FilesystemTableBackendDetails() .withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath("blobs") .withBasePath("blobs")
.withCardinality(Cardinality.ONE) .withCardinality(Cardinality.ONE)
.withFileNameFieldName("fileName")
.withContentsFieldName("contents")
); );
} }
@ -269,6 +268,8 @@ public class TestUtils
.withBackendDetails(new S3TableBackendDetails() .withBackendDetails(new S3TableBackendDetails()
.withBasePath("blobs") .withBasePath("blobs")
.withCardinality(Cardinality.ONE) .withCardinality(Cardinality.ONE)
.withFileNameFieldName("fileName")
.withContentsFieldName("contents")
); );
} }

195
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetailsTest.java Normal file
View File

@ -0,0 +1,195 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
import java.util.function.Consumer;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.fail;
/*******************************************************************************
** Unit test for AbstractFilesystemTableBackendDetails
*******************************************************************************/
class AbstractFilesystemTableBackendDetailsTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testValidInstancePasses() throws QInstanceValidationException
{
new QInstanceValidator().validate(QContext.getQInstance());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testMissingCardinality() throws QException
{
assertValidationFailureReasons((QInstance qInstance) ->
{
qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3).withBackendDetails(new FilesystemTableBackendDetails());
}, false, "missing cardinality");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testCardinalityOneIssues() throws QException
{
assertValidationFailureReasons((QInstance qInstance) ->
{
qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withBackendDetails(new FilesystemTableBackendDetails()
.withCardinality(Cardinality.ONE)
);
}, false, "missing contentsFieldName", "missing fileNameFieldName");
assertValidationFailureReasons((QInstance qInstance) ->
{
qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withBackendDetails(new FilesystemTableBackendDetails()
.withCardinality(Cardinality.ONE)
.withContentsFieldName("foo")
.withFileNameFieldName("bar")
);
}, false, "contentsFieldName [foo] is not a field", "fileNameFieldName [bar] is not a field");
assertValidationFailureReasons((QInstance qInstance) ->
{
qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withBackendDetails(new FilesystemTableBackendDetails()
.withCardinality(Cardinality.ONE)
.withContentsFieldName("contents")
.withFileNameFieldName("fileName")
.withRecordFormat(RecordFormat.CSV)
);
}, false, "has a recordFormat");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testCardinalityManyIssues() throws QException
{
assertValidationFailureReasons((QInstance qInstance) ->
{
qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_CSV).withBackendDetails(new FilesystemTableBackendDetails()
.withCardinality(Cardinality.MANY)
);
}, false, "missing recordFormat");
assertValidationFailureReasons((QInstance qInstance) ->
{
qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_CSV).withBackendDetails(new FilesystemTableBackendDetails()
.withCardinality(Cardinality.MANY)
.withRecordFormat(RecordFormat.CSV)
.withContentsFieldName("foo")
.withFileNameFieldName("bar")
);
}, false, "has a contentsFieldName", "has a fileNameFieldName");
}
/*******************************************************************************
** Implementation for the overloads of this name.
*******************************************************************************/
private void assertValidationFailureReasons(Consumer<QInstance> setup, boolean allowExtraReasons, String... reasons) throws QException
{
try
{
QInstance qInstance = TestUtils.defineInstance();
setup.accept(qInstance);
new QInstanceValidator().validate(qInstance);
fail("Should have thrown validationException");
}
catch(QInstanceValidationException e)
{
if(!allowExtraReasons)
{
int noOfReasons = e.getReasons() == null ? 0 : e.getReasons().size();
assertEquals(reasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", reasons)
+ "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", e.getReasons()) : "--"));
}
for(String reason : reasons)
{
assertReason(reason, e);
}
}
}
/*******************************************************************************
** Assert that an instance is valid!
*******************************************************************************/
private void assertValidationSuccess(Consumer<QInstance> setup) throws QException
{
try
{
QInstance qInstance = TestUtils.defineInstance();
setup.accept(qInstance);
new QInstanceValidator().validate(qInstance);
}
catch(QInstanceValidationException e)
{
fail("Expected no validation errors, but received: " + e.getMessage());
}
}
/*******************************************************************************
** utility method for asserting that a specific reason string is found within
** the list of reasons in the QInstanceValidationException.
**
*******************************************************************************/
private void assertReason(String reason, QInstanceValidationException e)
{
assertNotNull(e.getReasons(), "Expected there to be a reason for the failure (but there was not)");
assertThat(e.getReasons())
.withFailMessage("Expected any of:\n%s\nTo match: [%s]", e.getReasons(), reason)
.anyMatch(s -> s.contains(reason));
}
}

85
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java
View File

@ -25,6 +25,11 @@ package com.kingsrook.qqq.backend.module.filesystem.local;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -36,6 +41,8 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/******************************************************************************* /*******************************************************************************
@ -47,6 +54,9 @@ public class FilesystemBackendModuleTest
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach @BeforeEach
public void beforeEach() throws IOException public void beforeEach() throws IOException
{ {
@ -55,6 +65,9 @@ public class FilesystemBackendModuleTest
/*******************************************************************************
**
*******************************************************************************/
@AfterEach @AfterEach
public void afterEach() throws Exception public void afterEach() throws Exception
{ {
@ -63,6 +76,78 @@ public class FilesystemBackendModuleTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testListFiles() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
AbstractFilesystemAction abstractFilesystemAction = new AbstractFilesystemAction();
QBackendMetaData backend = qInstance.getBackendForTable(table.getName());
//////////////////////////////////////////////////////////
// with no filter given, all (3) files should come back //
//////////////////////////////////////////////////////////
List<File> files = abstractFilesystemAction.listFiles(table, backend);
assertEquals(3, files.size());
/////////////////////////////////////////
// filter for a file name that's found //
/////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
assertEquals(1, files.size());
assertEquals("BLOB-2.txt", files.get(0).getName());
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
assertEquals(1, files.size());
assertEquals("BLOB-1.txt", files.get(0).getName());
///////////////////////////////////
// filter for 2 names that exist //
///////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
assertEquals(2, files.size());
/////////////////////////////////////////////
// filter for a file name that isn't found //
/////////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
assertEquals(0, files.size());
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
assertEquals(1, files.size());
////////////////////////////////////////////////////
// 2 criteria, and'ed, and can't match, so find 0 //
////////////////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
assertEquals(0, files.size());
//////////////////////////////////////////////////
// 2 criteria, or'ed, and both match, so find 2 //
//////////////////////////////////////////////////
files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
.withBooleanOperator(QQueryFilter.BooleanOperator.OR));
assertEquals(2, files.size());
//////////////////////////////////////
// ensure unsupported filters throw //
//////////////////////////////////////
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
.hasMessageContaining("Unable to query filesystem table by field");
assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
.hasMessageContaining("Unable to query filename field using operator");
}
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/

32
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java
View File

@ -89,6 +89,7 @@ public class FilesystemActionTest extends BaseTest
writePersonJSONFiles(baseDirectory); writePersonJSONFiles(baseDirectory);
writePersonCSVFiles(baseDirectory); writePersonCSVFiles(baseDirectory);
writeBlobFiles(baseDirectory);
} }
@ -158,6 +159,37 @@ public class FilesystemActionTest extends BaseTest
/*******************************************************************************
** Write some data files into the directory for the filesystem module.
*******************************************************************************/
private void writeBlobFiles(File baseDirectory) throws IOException
{
String fullPath = baseDirectory.getAbsolutePath();
if(TestUtils.defineLocalFilesystemBlobTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
{
if(StringUtils.hasContent(details.getBasePath()))
{
fullPath += File.separatorChar + details.getBasePath();
}
}
fullPath += File.separatorChar;
String data1 = """
Hello, Blob
""";
FileUtils.writeStringToFile(new File(fullPath + "BLOB-1.txt"), data1);
String data2 = """
Hi Bob""";
FileUtils.writeStringToFile(new File(fullPath + "BLOB-2.txt"), data2);
String data3 = """
# Hi MD...""";
FileUtils.writeStringToFile(new File(fullPath + "BLOB-3.md"), data3);
}
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/

58
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java
View File

@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -32,8 +35,10 @@ import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractPostReadFileCustomizer; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractPostReadFileCustomizer;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.FilesystemTableCustomizers; import com.kingsrook.qqq.backend.module.filesystem.base.actions.FilesystemTableCustomizers;
import org.junit.jupiter.api.Assertions; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/******************************************************************************* /*******************************************************************************
@ -51,8 +56,8 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
QueryInput queryInput = new QueryInput(); QueryInput queryInput = new QueryInput();
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName()); queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput); QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
Assertions.assertTrue(queryOutput.getRecords().stream() assertTrue(queryOutput.getRecords().stream()
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)), .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
"All records should have a full-path in their backend details, matching the test folder name"); "All records should have a full-path in their backend details, matching the test folder name");
} }
@ -74,14 +79,57 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName()); queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput); QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
Assertions.assertTrue( assertTrue(
queryOutput.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")), queryOutput.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")),
"All records should have their email addresses up-shifted."); "All records should have their email addresses up-shifted.");
} }
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQueryForCardinalityOne() throws QException
{
FilesystemQueryAction filesystemQueryAction = new FilesystemQueryAction();
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
queryInput.setFilter(new QQueryFilter());
QueryOutput queryOutput = filesystemQueryAction.execute(queryInput);
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
queryOutput = filesystemQueryAction.execute(queryInput);
assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
////////////////////////////////////////////////////////////////
// put a glob on the table - now should only find 2 txt files //
////////////////////////////////////////////////////////////////
QInstance instance = TestUtils.defineInstance();
((FilesystemTableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).getBackendDetails()))
.withGlob("*.txt");
reInitInstanceInContext(instance);
queryInput.setFilter(new QQueryFilter());
queryOutput = filesystemQueryAction.execute(queryInput);
assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
//////////////////////////////
// add a limit to the query //
//////////////////////////////
queryInput.setFilter(new QQueryFilter().withLimit(1));
queryOutput = filesystemQueryAction.execute(queryInput);
assertEquals(1, queryOutput.getRecords().size(), "Query with limit should be respected");
}
/*******************************************************************************
**
*******************************************************************************/
public static class ValueUpshifter extends AbstractPostReadFileCustomizer public static class ValueUpshifter extends AbstractPostReadFileCustomizer
{ {
@Override @Override

14
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java
View File

@ -26,7 +26,7 @@ import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction; import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@ -38,6 +38,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModuleSubclassForTest; import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModuleSubclassForTest;
@ -187,7 +189,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/
private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QException
{ {
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend); S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase(); AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
@ -197,8 +199,8 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
for(String path : paths) for(String path : paths)
{ {
assertTrue(s3ObjectSummaries.stream().anyMatch(s3o -> s3o.getKey().equals(path)), assertTrue(s3ObjectSummaries.stream().anyMatch(s3o -> s3o.getKey().equals(path)),
"Path [" + path + "] should be in the listing, but was not. Full listing is: " + "Path [" + path + "] should be in the listing, but was not. Full listing is: "
s3ObjectSummaries.stream().map(S3ObjectSummary::getKey).collect(Collectors.joining(","))); + s3ObjectSummaries.stream().map(S3ObjectSummary::getKey).collect(Collectors.joining(",")));
} }
} }
@ -207,7 +209,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/
private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QException
{ {
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend); S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase(); AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
@ -257,6 +259,8 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
.withBackendName(backend.getName()) .withBackendName(backend.getName())
.withField(new QFieldMetaData("id", QFieldType.INTEGER)) .withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withBackendDetails(new S3TableBackendDetails() .withBackendDetails(new S3TableBackendDetails()
.withCardinality(Cardinality.MANY)
.withRecordFormat(RecordFormat.CSV)
.withBasePath(path) .withBasePath(path)
.withGlob(glob)); .withGlob(glob));
qInstance.addTable(qTableMetaData); qInstance.addTable(qTableMetaData);

4
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java
View File

@ -35,6 +35,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.BaseTest; import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
@ -118,6 +120,8 @@ class FilesystemSyncProcessTest extends BaseTest
.withBackendName(TestUtils.BACKEND_NAME_LOCAL_FS) .withBackendName(TestUtils.BACKEND_NAME_LOCAL_FS)
.withField(new QFieldMetaData("id", QFieldType.INTEGER)) .withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withBackendDetails(new FilesystemTableBackendDetails() .withBackendDetails(new FilesystemTableBackendDetails()
.withCardinality(Cardinality.MANY)
.withRecordFormat(RecordFormat.CSV)
.withBasePath(subPath)); .withBasePath(subPath));
} }

3
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java
View File

@ -62,6 +62,9 @@ public class BaseS3Test extends BaseTest
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2()); amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test"); amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3()); amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-1.txt", "Hello, Blob");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-2.txt", "Hi, Bob");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-3.md", "# Hi, MD");
} }

85
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java
View File

@ -25,6 +25,11 @@ package com.kingsrook.qqq.backend.module.filesystem.s3;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@ -32,6 +37,9 @@ import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemExceptio
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/******************************************************************************* /*******************************************************************************
@ -43,6 +51,83 @@ public class S3BackendModuleTest extends BaseS3Test
/*******************************************************************************
**
*******************************************************************************/
@Test
void testListFiles() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_S3);
QBackendMetaData backend = qInstance.getBackendForTable(table.getName());
//////////////////////////////////////////////////////
// set up the backend module (e.g., for localstack) //
//////////////////////////////////////////////////////
S3BackendModule s3BackendModule = new S3BackendModule();
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
actionBase.setS3Utils(getS3Utils());
//////////////////////////////////////////////////////////
// with no filter given, all (3) files should come back //
//////////////////////////////////////////////////////////
List<S3ObjectSummary> files = actionBase.listFiles(table, backend);
assertEquals(3, files.size());
/////////////////////////////////////////
// filter for a file name that's found //
/////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
assertEquals(1, files.size());
assertThat(files.get(0).getKey()).contains("BLOB-2.txt");
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
assertEquals(1, files.size());
assertThat(files.get(0).getKey()).contains("BLOB-1.txt");
///////////////////////////////////
// filter for 2 names that exist //
///////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
assertEquals(2, files.size());
/////////////////////////////////////////////
// filter for a file name that isn't found //
/////////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
assertEquals(0, files.size());
files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
assertEquals(1, files.size());
////////////////////////////////////////////////////
// 2 criteria, and'ed, and can't match, so find 0 //
////////////////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
assertEquals(0, files.size());
//////////////////////////////////////////////////
// 2 criteria, or'ed, and both match, so find 2 //
//////////////////////////////////////////////////
files = actionBase.listFiles(table, backend, new QQueryFilter(
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
.withBooleanOperator(QQueryFilter.BooleanOperator.OR));
assertEquals(2, files.size());
//////////////////////////////////////
// ensure unsupported filters throw //
//////////////////////////////////////
assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
.hasMessageContaining("Unable to query filesystem table by field");
assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
.hasMessageContaining("Unable to query filename field using operator");
}
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/

48
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java
View File

@ -23,13 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/******************************************************************************* /*******************************************************************************
@ -66,4 +72,46 @@ public class S3QueryActionTest extends BaseS3Test
return queryInput; return queryInput;
} }
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQueryForCardinalityOne() throws QException
{
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_S3);
queryInput.setFilter(new QQueryFilter());
S3QueryAction s3QueryAction = new S3QueryAction();
s3QueryAction.setS3Utils(getS3Utils());
QueryOutput queryOutput = s3QueryAction.execute(queryInput);
assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
queryOutput = s3QueryAction.execute(queryInput);
assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
////////////////////////////////////////////////////////////////
// put a glob on the table - now should only find 2 txt files //
////////////////////////////////////////////////////////////////
QInstance instance = TestUtils.defineInstance();
((S3TableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_S3).getBackendDetails()))
.withGlob("*.txt");
reInitInstanceInContext(instance);
queryInput.setFilter(new QQueryFilter());
queryOutput = s3QueryAction.execute(queryInput);
assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
//////////////////////////////
// add a limit to the query //
//////////////////////////////
queryInput.setFilter(new QQueryFilter().withLimit(1));
queryOutput = s3QueryAction.execute(queryInput);
assertEquals(1, queryOutput.getRecords().size(), "Query with limit should be respected");
}
} }

12
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java
View File

@ -22,10 +22,10 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.utils; package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.List; import java.util.List;
import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@ -42,14 +42,14 @@ public class S3UtilsTest extends BaseS3Test
** **
*******************************************************************************/ *******************************************************************************/
@Test @Test
public void testListObjectsInBucketAtPath() public void testListObjectsInBucketAtPath() throws QException
{ {
S3Utils s3Utils = getS3Utils(); S3Utils s3Utils = getS3Utils();
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders"); assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders");
assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders"); assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders"); assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders");
assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders"); assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders");
assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders"); assertEquals(7, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash"); assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash"); assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes"); assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes");
@ -60,8 +60,8 @@ public class S3UtilsTest extends BaseS3Test
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /"); assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s"); assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string"); assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string");
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively"); assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively"); assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
} }
@ -70,7 +70,7 @@ public class S3UtilsTest extends BaseS3Test
** **
*******************************************************************************/ *******************************************************************************/
@Test @Test
public void testGetObjectAsInputStream() throws IOException public void testGetObjectAsInputStream() throws Exception
{ {
S3Utils s3Utils = getS3Utils(); S3Utils s3Utils = getS3Utils();
List<S3ObjectSummary> s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", ""); List<S3ObjectSummary> s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", "");