QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

QQQ-14 add FileSyncProcess; Do globbing in s3; Refactoring module vs. base action

Browse Source
This commit is contained in:
darin.kelkhoff
2022-06-28 11:28:09 -05:00
parent daacfa6849
commit 1e376b9359
30 changed files with 1769 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -51,7 +51,7 @@
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-backend-core</artifactId>
<version>0.0.0-20220623.214704-10</version>
<version>0.0.0-20220628.161829-14</version>
</dependency>
<!-- 3rd party deps specifically for this module -->

22
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java
View File

@ -22,30 +22,18 @@
package com.kingsrook.qqq.backend.module.filesystem.base;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
/*******************************************************************************
** Interface to add additional functionality commonly among the various filesystem
** module implementations.
*******************************************************************************/
public interface FilesystemBackendModuleInterface
public interface FilesystemBackendModuleInterface<FILE>
{
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
** For filesystem backends, get the module-specific action base-class, that helps
** with functions like listing and deleting files.
*******************************************************************************/
void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException;
/*******************************************************************************
** Move a file from a source path, to a destination path.
**
** @throws FilesystemException if the move is known to have failed
*******************************************************************************/
void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException;
AbstractBaseFilesystemAction<FILE> getActionBase();
}

144
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
View File

@ -27,27 +27,42 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractQTableRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.NotImplementedException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Base class for all Filesystem actions across all modules.
**
** @param FILE The class that represents a file in the sub-module. e.g.,
* a java.io.File, or an S3Object.
*******************************************************************************/
public abstract class AbstractBaseFilesystemAction<FILE>
{
private static final Logger LOG = LogManager.getLogger(AbstractBaseFilesystemAction.class);
/*******************************************************************************
** List the files for a table - to be implemented in module-specific subclasses.
@ -60,31 +75,79 @@ public abstract class AbstractBaseFilesystemAction<FILE>
public abstract InputStream readFile(FILE file) throws IOException;
/*******************************************************************************
** Add backend details to records about the file that they are in.
** Write a file - to be implemented in module-specific subclasses.
*******************************************************************************/
protected abstract void addBackendDetailsToRecords(List<QRecord> recordsInFile, FILE file);
public abstract void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException;
/*******************************************************************************
** Get a string that represents the full path to a file.
*******************************************************************************/
protected abstract String getFullPathForFile(FILE file);
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
*******************************************************************************/
public abstract void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException;
/*******************************************************************************
** Move a file from a source path, to a destination path.
**
** @throws FilesystemException if the move is known to have failed
*******************************************************************************/
public abstract void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException;
/*******************************************************************************
** e.g., with a base path of /foo/
** and a table path of /bar/
** and a file at /foo/bar/baz.txt
** give us just the baz.txt part.
*******************************************************************************/
public abstract String stripBackendAndTableBasePathsFromFileName(FILE file, QBackendMetaData sourceBackend, QTableMetaData sourceTable);
/*******************************************************************************
** Append together the backend's base path (if present), with a table's path (again, if present).
** Append together the backend's base path (if present), with a table's base
** path (again, if present).
*******************************************************************************/
protected String getFullPath(QTableMetaData table, QBackendMetaData backendBase)
public String getFullBasePath(QTableMetaData table, QBackendMetaData backendBase)
{
AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase);
String fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : "";
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
if(StringUtils.hasContent(tableDetails.getPath()))
if(StringUtils.hasContent(tableDetails.getBasePath()))
{
fullPath += File.separatorChar + tableDetails.getPath();
fullPath += File.separatorChar + tableDetails.getBasePath();
}
fullPath += File.separatorChar;
fullPath = stripDuplicatedSlashes(fullPath);
return fullPath;
}
/*******************************************************************************
**
*******************************************************************************/
public static String stripDuplicatedSlashes(String path)
{
if(path == null)
{
return (null);
}
return (path.replaceAll("//+", "/"));
}
/*******************************************************************************
** Get the backend metaData, type-checked as the requested type.
*******************************************************************************/
@ -119,6 +182,8 @@ public abstract class AbstractBaseFilesystemAction<FILE>
*******************************************************************************/
public QueryResult executeQuery(QueryRequest queryRequest) throws QException
{
preAction(queryRequest);
try
{
QueryResult rs = new QueryResult();
@ -131,20 +196,25 @@ public abstract class AbstractBaseFilesystemAction<FILE>
for(FILE file : files)
{
LOG.info("Processing file: " + getFullPathForFile(file));
switch(tableDetails.getRecordFormat())
{
case "csv":
case CSV:
{
String fileContents = IOUtils.toString(readFile(file));
fileContents = customizeFileContentsAfterReading(table, fileContents);
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
records.addAll(recordsInFile);
break;
}
case "json":
case JSON:
{
String fileContents = IOUtils.toString(readFile(file));
fileContents = customizeFileContentsAfterReading(table, fileContents);
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
@ -162,8 +232,64 @@ public abstract class AbstractBaseFilesystemAction<FILE>
}
catch(Exception e)
{
e.printStackTrace();
LOG.warn("Error executing query", e);
throw new QException("Error executing query", e);
}
}
/*******************************************************************************
** Add backend details to records about the file that they are in.
*******************************************************************************/
protected void addBackendDetailsToRecords(List<QRecord> recordsInFile, FILE file)
{
recordsInFile.forEach(record ->
{
record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, getFullPathForFile(file));
});
}
/*******************************************************************************
** Method that subclasses can override to add pre-action things (e.g., setting up
** s3 client).
*******************************************************************************/
protected void preAction(AbstractQTableRequest tableRequest)
{
/////////////////////////////////////////////////////////////////////
// noop in base class - subclasses can add functionality if needed //
/////////////////////////////////////////////////////////////////////
}
/*******************************************************************************
**
*******************************************************************************/
private String customizeFileContentsAfterReading(QTableMetaData table, String fileContents) throws QException
{
Optional<QCodeReference> optionalCustomizer = table.getCustomizer("postFileRead");
if(optionalCustomizer.isEmpty())
{
return (fileContents);
}
QCodeReference customizer = optionalCustomizer.get();
try
{
Class<?> customizerClass = Class.forName(customizer.getName());
@SuppressWarnings("unchecked")
Function<String, String> function = (Function<String, String>) customizerClass.getConstructor().newInstance();
return function.apply(fileContents);
}
catch(Exception e)
{
throw (new QException("Error customizing file contents", e));
}
}
}

72
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java
View File

@ -30,42 +30,78 @@ import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
*******************************************************************************/
public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
{
private String path;
private String recordFormat; // todo - enum? but hard w/ serialization?
private String cardinality; // todo - enum?
private String basePath;
private String glob;
private RecordFormat recordFormat;
private Cardinality cardinality;
/*******************************************************************************
** Getter for path
** Getter for basePath
**
*******************************************************************************/
public String getPath()
public String getBasePath()
{
return path;
return basePath;
}
/*******************************************************************************
** Setter for path
** Setter for basePath
**
*******************************************************************************/
public void setPath(String path)
public void setBasePath(String basePath)
{
this.path = path;
this.basePath = basePath;
}
/*******************************************************************************
** Fluent Setter for path
** Fluent Setter for basePath
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends AbstractFilesystemTableBackendDetails> T withPath(String path)
public <T extends AbstractFilesystemTableBackendDetails> T withBasePath(String basePath)
{
this.path = path;
this.basePath = basePath;
return (T) this;
}
/*******************************************************************************
** Getter for glob
**
*******************************************************************************/
public String getGlob()
{
return glob;
}
/*******************************************************************************
** Setter for glob
**
*******************************************************************************/
public void setGlob(String glob)
{
this.glob = glob;
}
/*******************************************************************************
** Fluent Setter for glob
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends AbstractFilesystemTableBackendDetails> T withGlob(String glob)
{
this.glob = glob;
return (T) this;
}
@ -75,7 +111,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
** Getter for recordFormat
**
*******************************************************************************/
public String getRecordFormat()
public RecordFormat getRecordFormat()
{
return recordFormat;
}
@ -86,7 +122,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
** Setter for recordFormat
**
*******************************************************************************/
public void setRecordFormat(String recordFormat)
public void setRecordFormat(RecordFormat recordFormat)
{
this.recordFormat = recordFormat;
}
@ -98,7 +134,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends AbstractFilesystemTableBackendDetails> T withRecordFormat(String recordFormat)
public <T extends AbstractFilesystemTableBackendDetails> T withRecordFormat(RecordFormat recordFormat)
{
this.recordFormat = recordFormat;
return ((T) this);
@ -110,7 +146,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
** Getter for cardinality
**
*******************************************************************************/
public String getCardinality()
public Cardinality getCardinality()
{
return cardinality;
}
@ -121,7 +157,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
** Setter for cardinality
**
*******************************************************************************/
public void setCardinality(String cardinality)
public void setCardinality(Cardinality cardinality)
{
this.cardinality = cardinality;
}
@ -133,7 +169,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends AbstractFilesystemTableBackendDetails> T withCardinality(String cardinality)
public <T extends AbstractFilesystemTableBackendDetails> T withCardinality(Cardinality cardinality)
{
this.cardinality = cardinality;
return ((T) this);

33
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/Cardinality.java Normal file
View File

@ -0,0 +1,33 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
/*******************************************************************************
** Mode for filesystem backends: are all records in a single file, or are there
** many files?
*******************************************************************************/
public enum Cardinality
{
ONE,
MANY
}

34
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/RecordFormat.java Normal file
View File

@ -0,0 +1,34 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
/*******************************************************************************
** How are records stored in the files in a filesystem backend? CSV, JSON,
** future types may XML, or more exotic ones, like "BINARY" or "TEXT" (e.g., 1
** record and 1 field per-file)
*******************************************************************************/
public enum RecordFormat
{
CSV,
JSON
}

84
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java
View File

@ -24,16 +24,15 @@ package com.kingsrook.qqq.backend.module.filesystem.local;
import java.io.File;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction;
@ -52,6 +51,19 @@ public class FilesystemBackendModule implements QBackendModuleInterface, Filesys
private static final Logger LOG = LogManager.getLogger(FilesystemBackendModule.class);
/*******************************************************************************
** For filesystem backends, get the module-specific action base-class, that helps
** with functions like listing and deleting files.
*******************************************************************************/
@Override
public AbstractBaseFilesystemAction<File> getActionBase()
{
return (new AbstractFilesystemAction());
}
/*******************************************************************************
** Method where a backend module must be able to provide its type (name).
*******************************************************************************/
@ -127,70 +139,4 @@ public class FilesystemBackendModule implements QBackendModuleInterface, Filesys
return (new FilesystemDeleteAction());
}
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
*******************************************************************************/
@Override
public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
{
File file = new File(fileReference);
if(!file.exists())
{
//////////////////////////////////////////////////////////////////////////////////////////////
// if the file doesn't exist, just exit with noop. don't throw an error - that should only //
// happen if the "contract" of the method is broken, and the file still exists //
//////////////////////////////////////////////////////////////////////////////////////////////
LOG.debug("Not deleting file [{}], because it does not exist.", file);
return;
}
if(!file.delete())
{
throw (new FilesystemException("Failed to delete file: " + fileReference));
}
}
/*******************************************************************************
** Move a file from a source path, to a destination path.
**
** @throws FilesystemException if the delete is known to have failed
*******************************************************************************/
@Override
public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException
{
File sourceFile = new File(source);
File destinationFile = new File(destination);
File destinationParent = destinationFile.getParentFile();
if(!sourceFile.exists())
{
throw (new FilesystemException("Cannot move file " + source + ", as it does not exist."));
}
//////////////////////////////////////////////////////////////////////////////////////
// if the destination folder doesn't exist, try to make it - and fail if that fails //
//////////////////////////////////////////////////////////////////////////////////////
if(!destinationParent.exists())
{
LOG.debug("Making destination directory {} for move", destinationParent.getAbsolutePath());
if(!destinationParent.mkdirs())
{
throw (new FilesystemException("Failed to make destination directory " + destinationParent.getAbsolutePath() + " to move " + source + " into."));
}
}
if(!sourceFile.renameTo(destinationFile))
{
throw (new FilesystemException("Failed to move (rename) file " + source + " to " + destination));
}
}
}

114
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java
View File

@ -29,11 +29,14 @@ import java.io.InputStream;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
@ -41,6 +44,9 @@ import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFile
*******************************************************************************/
public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
{
private static final Logger LOG = LogManager.getLogger(AbstractFilesystemAction.class);
/*******************************************************************************
** List the files for this table.
@ -48,7 +54,8 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
@Override
public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase)
{
String fullPath = getFullPath(table, backendBase);
// todo - needs rewritten to do globbing...
String fullPath = getFullBasePath(table, backendBase);
File directory = new File(fullPath);
File[] files = directory.listFiles();
@ -74,15 +81,106 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
/*******************************************************************************
** Add backend details to records about the file that they are in.
** Write a file - to be implemented in module-specific subclasses.
*******************************************************************************/
@Override
protected void addBackendDetailsToRecords(List<QRecord> recordsInFile, File file)
public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
{
recordsInFile.forEach(record ->
FileUtils.writeByteArrayToFile(new File(path), contents);
}
/*******************************************************************************
** Get a string that represents the full path to a file.
*******************************************************************************/
@Override
protected String getFullPathForFile(File file)
{
record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, file.getAbsolutePath());
});
return (file.getAbsolutePath());
}
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
*******************************************************************************/
@Override
public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
{
File file = new File(fileReference);
if(!file.exists())
{
//////////////////////////////////////////////////////////////////////////////////////////////
// if the file doesn't exist, just exit with noop. don't throw an error - that should only //
// happen if the "contract" of the method is broken, and the file still exists //
//////////////////////////////////////////////////////////////////////////////////////////////
LOG.debug("Not deleting file [{}], because it does not exist.", file);
return;
}
if(!file.delete())
{
throw (new FilesystemException("Failed to delete file: " + fileReference));
}
}
/*******************************************************************************
** Move a file from a source path, to a destination path.
**
** @param destination assumed to be a file path - not a directory
** @throws FilesystemException if the delete is known to have failed
*******************************************************************************/
@Override
public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException
{
File sourceFile = new File(source);
File destinationFile = new File(destination);
File destinationParent = destinationFile.getParentFile();
if(!sourceFile.exists())
{
throw (new FilesystemException("Cannot move file " + source + ", as it does not exist."));
}
//////////////////////////////////////////////////////////////////////////////////////
// if the destination folder doesn't exist, try to make it - and fail if that fails //
//////////////////////////////////////////////////////////////////////////////////////
if(!destinationParent.exists())
{
LOG.debug("Making destination directory {} for move", destinationParent.getAbsolutePath());
if(!destinationParent.mkdirs())
{
throw (new FilesystemException("Failed to make destination directory " + destinationParent.getAbsolutePath() + " to move " + source + " into."));
}
}
if(!sourceFile.renameTo(destinationFile))
{
throw (new FilesystemException("Failed to move (rename) file " + source + " to " + destination));
}
}
/*******************************************************************************
** e.g., with a base path of /foo/
** and a table path of /bar/
** and a file at /foo/bar/baz.txt
** give us just the baz.txt part.
*******************************************************************************/
@Override
public String stripBackendAndTableBasePathsFromFileName(File file, QBackendMetaData backend, QTableMetaData table)
{
String tablePath = getFullBasePath(table, backend);
String strippedPath = file.getAbsolutePath().replaceFirst(".*" + tablePath, "");
return (strippedPath);
}
}

40
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java
View File

@ -23,8 +23,6 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et
import java.io.File;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
@ -43,7 +41,8 @@ import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
@ -52,11 +51,20 @@ import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendD
*******************************************************************************/
public class BasicETLCleanupSourceFilesFunction implements FunctionBody
{
private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesFunction.class);
public static final String FIELD_MOVE_OR_DELETE = "moveOrDelete";
public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves";
public static final String VALUE_MOVE = "move";
public static final String VALUE_DELETE = "delete";
public static final String FUNCTION_NAME = "cleanupSourceFiles";
/*******************************************************************************
** Execute the function - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
{
@ -70,17 +78,22 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function."));
}
Set<String> sourceFiles = runFunctionRequest.getRecords().stream()
.map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))
.collect(Collectors.toSet());
String sourceFilePaths = runFunctionRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS);
if(!StringUtils.hasContent(sourceFilePaths))
{
LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS + "]");
return;
}
String[] sourceFiles = sourceFilePaths.split(",");
for(String sourceFile : sourceFiles)
{
String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE);
if("delete".equals(moveOrDelete))
if(VALUE_DELETE.equals(moveOrDelete))
{
filesystemModule.deleteFile(runFunctionRequest.getInstance(), table, sourceFile);
filesystemModule.getActionBase().deleteFile(runFunctionRequest.getInstance(), table, sourceFile);
}
else if("move".equals(moveOrDelete))
else if(VALUE_MOVE.equals(moveOrDelete))
{
String destinationForMoves = runFunctionRequest.getValueString(FIELD_DESTINATION_FOR_MOVES);
if(!StringUtils.hasContent(destinationForMoves))
@ -89,11 +102,12 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
}
File file = new File(sourceFile);
String destinationPath = destinationForMoves + File.separator + file.getName();
filesystemModule.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath);
filesystemModule.getActionBase().moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath);
}
else
{
throw (new QException("Unexpected value [" + moveOrDelete + "] for field [" + FIELD_MOVE_OR_DELETE + "]. Must be either [move] or [delete]."));
throw (new QException("Unexpected value [" + moveOrDelete + "] for field [" + FIELD_MOVE_OR_DELETE + "]. "
+ "Must be either [" + VALUE_MOVE + "] or [" + VALUE_DELETE + "]."));
}
}
}
@ -101,12 +115,12 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
/*******************************************************************************
**
** define the metaData that describes this function
*******************************************************************************/
public QFunctionMetaData defineFunctionMetaData()
{
return (new QFunctionMetaData()
.withName("cleanupSourceFiles")
.withName(FUNCTION_NAME)
.withCode(new QCodeReference()
.withName(this.getClass().getName())
.withCodeType(QCodeType.JAVA)

84
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java Normal file
View File

@ -0,0 +1,84 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
/*******************************************************************************
** Function body for collecting the file names that were discovered in the
** Extract step. These will be lost during the transform, so we capture them here,
** so that our Clean function can move or delete them.
**
** TODO - need unit test!!
*******************************************************************************/
public class BasicETLCollectSourceFileNamesFunction implements FunctionBody
{
public static final String FUNCTION_NAME = "collectSourceFileNames";
public static final String FIELD_SOURCE_FILE_PATHS = "sourceFilePaths";
/*******************************************************************************
** Execute the function - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
{
Set<String> sourceFiles = runFunctionRequest.getRecords().stream()
.map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))
.collect(Collectors.toSet());
runFunctionResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles));
}
/*******************************************************************************
** define the metaData that describes this function
*******************************************************************************/
public QFunctionMetaData defineFunctionMetaData()
{
return (new QFunctionMetaData()
.withName(FUNCTION_NAME)
.withCode(new QCodeReference()
.withName(this.getClass().getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.FUNCTION))
.withOutputMetaData(new QFunctionOutputMetaData()
.addField(new QFieldMetaData(FIELD_SOURCE_FILE_PATHS, QFieldType.STRING))));
}
}

123
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java Normal file
View File

@ -0,0 +1,123 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync;
import java.io.File;
import java.io.InputStream;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Function body for collecting the file names that were discovered in the
** Extract step. These will be lost during the transform, so we capture them here,
** so that our Clean function can move or delete them.
**
*******************************************************************************/
public class FilesystemSyncFunction implements FunctionBody
{
private static final Logger LOG = LogManager.getLogger(FilesystemSyncFunction.class);
public static final String FUNCTION_NAME = "sync";
/*******************************************************************************
** Execute the function - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
{
QTableMetaData sourceTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
QTableMetaData archiveTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName());
FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
Map<String, Object> sourceFiles = getFileNames(sourceModule.getActionBase(), sourceTable, sourceBackend);
QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName());
FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
Set<String> archiveFiles = getFileNames(archiveModule.getActionBase(), archiveTable, archiveBackend).keySet();
QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName());
FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet())
{
try
{
String sourceFileName = sourceEntry.getKey();
if(!archiveFiles.contains(sourceFileName))
{
LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]");
InputStream inputStream = sourceModule.getActionBase().readFile(sourceEntry.getValue());
byte[] bytes = inputStream.readAllBytes();
String archivePath = archiveModule.getActionBase().getFullBasePath(archiveTable, archiveBackend);
archiveModule.getActionBase().writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
String processingPath = processingModule.getActionBase().getFullBasePath(processingTable, processingBackend);
processingModule.getActionBase().writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
}
}
catch(Exception e)
{
LOG.error("Error processing file: " + sourceEntry, e);
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private Map<String, Object> getFileNames(AbstractBaseFilesystemAction actionBase, QTableMetaData table, QBackendMetaData backend)
{
List<Object> files = actionBase.listFiles(table, backend);
Map<String, Object> rs = new LinkedHashMap<>();
for(Object file : files)
{
String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(file, backend, table);
rs.put(fileName, file);
}
return (rs);
}
}

78
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java Normal file
View File

@ -0,0 +1,78 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
/*******************************************************************************
** Definition for Filesystem sync process.
**
** Job is to:
** - list all files in the source table.
** - list all files in the archive table.
** - if any files exist in the source, but not in the archive, then:
** - copy the file to both the archive and the processing table.
**
** The idea being, that the source is read-only, and we want to move files out of
** processing after they've been processed - and the archive is what we can have
** in-between the two.
*******************************************************************************/
public class FilesystemSyncProcess
{
public static final String PROCESS_NAME = "filesystem.sync";
public static final String FIELD_SOURCE_TABLE = "sourceTable";
public static final String FIELD_ARCHIVE_TABLE = "archiveTable";
public static final String FIELD_PROCESSING_TABLE = "processingTable";
/*******************************************************************************
**
*******************************************************************************/
public QProcessMetaData defineProcessMetaData()
{
QFunctionMetaData syncFunction = new QFunctionMetaData()
.withName(FilesystemSyncFunction.FUNCTION_NAME)
.withCode(new QCodeReference()
.withName(FilesystemSyncFunction.class.getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.FUNCTION))
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING))
.addField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING))
.addField(new QFieldMetaData(FIELD_PROCESSING_TABLE, QFieldType.STRING)));
return new QProcessMetaData()
.withName(PROCESS_NAME)
.addFunction(syncFunction);
}
}

81
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java
View File

@ -22,24 +22,23 @@
package com.kingsrook.qqq.backend.module.filesystem.s3;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
/*******************************************************************************
@ -47,7 +46,17 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
*******************************************************************************/
public class S3BackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface
{
private S3Utils s3Utils;
/*******************************************************************************
** For filesystem backends, get the module-specific action base-class, that helps
** with functions like listing and deleting files.
*******************************************************************************/
@Override
public AbstractBaseFilesystemAction<S3ObjectSummary> getActionBase()
{
return (new AbstractS3Action());
}
@ -84,43 +93,6 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
*******************************************************************************/
@Override
public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
{
QBackendMetaData backend = instance.getBackend(table.getBackendName());
String bucketName = ((S3BackendMetaData) backend).getBucketName();
getS3Utils().deleteObject(bucketName, fileReference);
}
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @throws FilesystemException if the move is known to have failed
*******************************************************************************/
@Override
public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException
{
QBackendMetaData backend = instance.getBackend(table.getBackendName());
String bucketName = ((S3BackendMetaData) backend).getBucketName();
getS3Utils().moveObject(bucketName, source, destination);
}
/*******************************************************************************
**
*******************************************************************************/
@ -163,29 +135,4 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke
return (new S3DeleteAction());
}
/*******************************************************************************
** Setter for s3Utils
*******************************************************************************/
public void setS3Utils(S3Utils s3Utils)
{
this.s3Utils = s3Utils;
}
/*******************************************************************************
** Internal accessor for the s3Utils object - should always use this, not the field.
*******************************************************************************/
private S3Utils getS3Utils()
{
if(s3Utils == null)
{
s3Utils = new S3Utils();
}
return s3Utils;
}
}

128
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java
View File

@ -25,14 +25,22 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.actions.AbstractQTableRequest;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
@ -40,10 +48,46 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
*******************************************************************************/
public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSummary>
{
private static final Logger LOG = LogManager.getLogger(AbstractS3Action.class);
private S3Utils s3Utils;
/*******************************************************************************
** Setup the s3 utils object to be used for this action.
*******************************************************************************/
@Override
protected void preAction(AbstractQTableRequest tableRequest)
{
super.preAction(tableRequest);
if(s3Utils != null)
{
LOG.debug("s3Utils object is already set - not re-setting it.");
return;
}
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, tableRequest.getBackend());
AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard();
if(StringUtils.hasContent(s3BackendMetaData.getAccessKey()) && StringUtils.hasContent(s3BackendMetaData.getSecretKey()))
{
BasicAWSCredentials credentials = new BasicAWSCredentials(s3BackendMetaData.getAccessKey(), s3BackendMetaData.getSecretKey());
clientBuilder.setCredentials(new AWSStaticCredentialsProvider(credentials));
}
if(StringUtils.hasContent(s3BackendMetaData.getRegion()))
{
clientBuilder.setRegion(s3BackendMetaData.getRegion());
}
s3Utils = new S3Utils();
s3Utils.setAmazonS3(clientBuilder.build());
}
/*******************************************************************************
** Set the S3Utils object.
*******************************************************************************/
@ -76,16 +120,16 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase)
{
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase);
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
String fullPath = getFullPath(table, backendBase);
String fullPath = getFullBasePath(table, backendBase);
String bucketName = s3BackendMetaData.getBucketName();
String glob = tableDetails.getGlob();
////////////////////////////////////////////////////////////////////
// todo - read metadata to decide if we should include subfolders //
// todo - look at metadata to configure the s3 client here? //
////////////////////////////////////////////////////////////////////
boolean includeSubfolders = false;
return getS3Utils().listObjectsInBucketAtPath(bucketName, fullPath, includeSubfolders);
return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob);
}
@ -102,15 +146,77 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
/*******************************************************************************
** Add backend details to records about the file that they are in.
** Write a file - to be implemented in module-specific subclasses.
*******************************************************************************/
@Override
protected void addBackendDetailsToRecords(List<QRecord> recordsInFile, S3ObjectSummary s3ObjectSummary)
public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException
{
recordsInFile.forEach(record ->
path = stripDuplicatedSlashes(path);
String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName();
getS3Utils().writeFile(bucketName, path, contents);
}
/*******************************************************************************
** Get a string that represents the full path to a file.
*******************************************************************************/
@Override
protected String getFullPathForFile(S3ObjectSummary s3ObjectSummary)
{
record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, s3ObjectSummary.getKey());
});
return (s3ObjectSummary.getKey());
}
/*******************************************************************************
** e.g., with a base path of /foo/
** and a table path of /bar/
** and a file at /foo/bar/baz.txt
** give us just the baz.txt part.
*******************************************************************************/
@Override
public String stripBackendAndTableBasePathsFromFileName(S3ObjectSummary file, QBackendMetaData backend, QTableMetaData table)
{
String tablePath = getFullBasePath(table, backend);
String strippedPath = file.getKey().replaceFirst(".*" + tablePath, "");
return (strippedPath);
}
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
*******************************************************************************/
@Override
public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
{
QBackendMetaData backend = instance.getBackend(table.getBackendName());
String bucketName = ((S3BackendMetaData) backend).getBucketName();
getS3Utils().deleteObject(bucketName, fileReference);
}
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @param destination assumed to be a file path - not a directory
** @throws FilesystemException if the move is known to have failed
*******************************************************************************/
@Override
public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException
{
QBackendMetaData backend = instance.getBackend(table.getBackendName());
String bucketName = ((S3BackendMetaData) backend).getBucketName();
getS3Utils().moveObject(bucketName, source, destination);
}
}

126
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java
View File

@ -22,16 +22,20 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
import com.kingsrook.qqq.backend.core.model.metadata.QSecretReader;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
/*******************************************************************************
** (local) Filesystem backend meta data.
** S3 backend meta data.
*******************************************************************************/
public class S3BackendMetaData extends AbstractFilesystemBackendMetaData
{
private String bucketName;
private String accessKey;
private String secretKey;
private String region;
@ -79,4 +83,124 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData
return (T) this;
}
/*******************************************************************************
** Getter for accessKey
**
*******************************************************************************/
public String getAccessKey()
{
return accessKey;
}
/*******************************************************************************
** Setter for accessKey
**
*******************************************************************************/
public void setAccessKey(String accessKey)
{
this.accessKey = accessKey;
}
/*******************************************************************************
** Fluent setter for accessKey
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends S3BackendMetaData> T withAccessKey(String accessKey)
{
this.accessKey = accessKey;
return (T) this;
}
/*******************************************************************************
** Getter for secretKey
**
*******************************************************************************/
public String getSecretKey()
{
return secretKey;
}
/*******************************************************************************
** Setter for secretKey
**
*******************************************************************************/
public void setSecretKey(String secretKey)
{
this.secretKey = secretKey;
}
/*******************************************************************************
** Fluent setter for secretKey
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends S3BackendMetaData> T withSecretKey(String secretKey)
{
this.secretKey = secretKey;
return (T) this;
}
/*******************************************************************************
** Getter for region
**
*******************************************************************************/
public String getRegion()
{
return region;
}
/*******************************************************************************
** Setter for region
**
*******************************************************************************/
public void setRegion(String region)
{
this.region = region;
}
/*******************************************************************************
** Fluent setter for region
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends S3BackendMetaData> T withRegion(String region)
{
this.region = region;
return (T) this;
}
/*******************************************************************************
** Called by the QInstanceEnricher - to do backend-type-specific enrichments.
** Original use case is: reading secrets into fields (e.g., passwords).
*******************************************************************************/
@Override
public void enrich()
{
super.enrich();
QSecretReader secretReader = new QSecretReader();
accessKey = secretReader.readSecret(accessKey);
secretKey = secretReader.readSecret(secretKey);
}
}

111
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java
View File

@ -22,7 +22,12 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.net.URI;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.util.ArrayList;
import java.util.List;
import com.amazonaws.regions.Regions;
@ -30,8 +35,10 @@ import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -47,25 +54,57 @@ public class S3Utils
{
private static final Logger LOG = LogManager.getLogger(S3Utils.class);
private AmazonS3 s3;
private AmazonS3 amazonS3;
/*******************************************************************************
** List the objects in an S3 bucket at a given path
** List the objects in an S3 bucket matching a glob, per:
** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
*******************************************************************************/
public List<S3ObjectSummary> listObjectsInBucketAtPath(String bucketName, String fullPath, boolean includeSubfolders)
public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob)
{
//////////////////////////////////////////////////////////////////////////////////////////////////
// s3 list requests find nothing if the path starts with a /, so strip away any leading slashes //
// also strip away trailing /'s, for consistent known paths. //
// also normalize any duplicated /'s to a single /. //
//////////////////////////////////////////////////////////////////////////////////////////////////
fullPath = fullPath.replaceFirst("^/+", "").replaceFirst("/+$", "").replaceAll("//+", "/");
path = path.replaceFirst("^/+", "").replaceFirst("/+$", "").replaceAll("//+", "/");
String prefix = path;
// todo - maybe this is some error - that the user put a * in the path instead of the glob?
if(prefix.indexOf('*') > -1)
{
prefix = prefix.substring(0, prefix.indexOf('*'));
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob. //
// That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. //
// Anyway, add a trailing /* to globs with no wildcards (or just a '*' if it's a request for the root ("")) //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(glob == null)
{
glob = "";
}
if(!glob.contains("*"))
{
if(glob.equals(""))
{
glob += "*";
}
else
{
glob += "/*";
}
}
String pathMatcherArg = AbstractFilesystemAction.stripDuplicatedSlashes("glob:/" + path + "/" + glob);
PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher(pathMatcherArg);
ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request()
.withBucketName(bucketName)
.withPrefix(fullPath);
.withPrefix(prefix);
ListObjectsV2Result listObjectsV2Result = null;
List<S3ObjectSummary> rs = new ArrayList<>();
@ -76,7 +115,8 @@ public class S3Utils
{
listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken());
}
listObjectsV2Result = getS3().listObjectsV2(listObjectsV2Request);
LOG.info("Listing bucket=" + bucketName + ", path=" + path);
listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request);
//////////////////////////////////
// put files in the result list //
@ -85,27 +125,30 @@ public class S3Utils
{
String key = objectSummary.getKey();
//////////////////
// skip folders //
//////////////////
//////////////////////////////////////////////////////////////////////////////////////////////////////
// it looks like keys in s3 can have duplicated /'s - so normalize those, to create a "sane" result //
//////////////////////////////////////////////////////////////////////////////////////////////////////
key = key.replaceAll("//+", "/");
////////////////////////////////////////////////////////////////////////////////
// always skip folders //
// this seemed to fire when it was first written, but not in our unit tests - //
// is this a difference with real s3 vs. localstack possibly? //
////////////////////////////////////////////////////////////////////////////////
if(key.endsWith("/"))
{
LOG.debug("Skipping file [{}] because it is a folder", key);
continue;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if we're not supposed to include subfolders, check the path on this file, and only include it if it matches the request //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(!includeSubfolders)
///////////////////////////////////////////
// skip files that do not match the glob //
///////////////////////////////////////////
if(!pathMatcher.matches(Path.of(URI.create("file:///" + key))))
{
String prefix = key.substring(0, key.lastIndexOf("/"));
if(!prefix.equals(fullPath))
{
LOG.debug("Skipping file [{}] in a sub-folder [{}] != [{}]", key, prefix, fullPath);
LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob);
continue;
}
}
rs.add(objectSummary);
}
@ -122,7 +165,17 @@ public class S3Utils
*******************************************************************************/
public InputStream getObjectAsInputStream(S3ObjectSummary s3ObjectSummary)
{
return getS3().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()).getObjectContent();
return getAmazonS3().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()).getObjectContent();
}
/*******************************************************************************
** Write a file
*******************************************************************************/
public void writeFile(String bucket, String key, byte[] contents)
{
getAmazonS3().putObject(bucket, key, new ByteArrayInputStream(contents), new ObjectMetadata());
}
@ -137,7 +190,7 @@ public class S3Utils
//////////////////////////////////////////////////////////////////////////////////////////////
try
{
getS3().deleteObject(bucketName, key);
getAmazonS3().deleteObject(bucketName, key);
}
catch(Exception e)
{
@ -157,8 +210,8 @@ public class S3Utils
//////////////////////////////////////////////////////////////////////////////////////////////
try
{
getS3().copyObject(bucketName, source, bucketName, destination);
getS3().deleteObject(bucketName, source);
getAmazonS3().copyObject(bucketName, source, bucketName, destination);
getAmazonS3().deleteObject(bucketName, source);
}
catch(Exception e)
{
@ -171,9 +224,9 @@ public class S3Utils
/*******************************************************************************
** Setter for AmazonS3 client object.
*******************************************************************************/
public void setAmazonS3(AmazonS3 s3)
public void setAmazonS3(AmazonS3 amazonS3)
{
this.s3 = s3;
this.amazonS3 = amazonS3;
}
@ -181,14 +234,14 @@ public class S3Utils
/*******************************************************************************
** Getter for AmazonS3 client object.
*******************************************************************************/
public AmazonS3 getS3()
public AmazonS3 getAmazonS3()
{
if(s3 == null)
if(amazonS3 == null)
{
s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build();
// TODO - get this (and other props?) from backend meta data
amazonS3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build();
}
return s3;
return amazonS3;
}
}

20
src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -33,6 +33,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.mock.MockAuthenticationModule;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
@ -56,6 +58,10 @@ public class TestUtils
///////////////////////////////////////////////////////////////////
public static final String BASE_PATH = "/tmp/filesystem-tests";
//////////////////////////////////////////////////////////////////////////////
// Used to make each test method have a unique folder path, more or less... //
// See methods that work with it. //
//////////////////////////////////////////////////////////////////////////////
private static int testInstanceCounter = 0;
@ -110,7 +116,7 @@ public class TestUtils
QInstance qInstance = new QInstance();
qInstance.setAuthentication(defineAuthentication());
qInstance.addBackend(defineLocalFilesystemBackend());
qInstance.addTable(defineLocalFilesystemCSVPersonTable());
qInstance.addTable(defineLocalFilesystemJSONPersonTable());
qInstance.addBackend(defineS3Backend());
qInstance.addTable(defineS3CSVPersonTable());
@ -149,7 +155,7 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineLocalFilesystemCSVPersonTable()
public static QTableMetaData defineLocalFilesystemJSONPersonTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_PERSON_LOCAL_FS)
@ -164,9 +170,9 @@ public class TestUtils
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
.withField(new QFieldMetaData("email", QFieldType.STRING))
.withBackendDetails(new FilesystemTableBackendDetails()
.withPath("persons")
.withRecordFormat("json")
.withCardinality("many")
.withBasePath("persons")
.withRecordFormat(RecordFormat.JSON)
.withCardinality(Cardinality.MANY)
);
}
@ -203,8 +209,8 @@ public class TestUtils
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
.withField(new QFieldMetaData("email", QFieldType.STRING))
.withBackendDetails(new S3TableBackendDetails()
.withRecordFormat("csv")
.withCardinality("many")
.withRecordFormat(RecordFormat.CSV)
.withCardinality(Cardinality.MANY)
);
}

13
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java
View File

@ -78,7 +78,7 @@ public class FilesystemBackendModuleTest
List<File> filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
filesystemBackendModule.deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath());
filesystemBackendModule.getActionBase().deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath());
List<File> filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeDelete.size() - 1, filesAfterDelete.size(),
@ -98,12 +98,13 @@ public class FilesystemBackendModuleTest
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
// note, we'd like to detect the non-delete, but there's no such info back from aws it appears? //
// note, our contract is that as long as the file doesn't exist after calling delete (e.g., if it wasn't there //
// to begin with, then we're okay, and don't expect an exception //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<File> filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
filesystemBackendModule.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST);
filesystemBackendModule.getActionBase().deleteFile(qInstance, table, PATH_THAT_WONT_EXIST);
List<File> filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeDelete.size(), filesAfterDelete.size(),
@ -133,7 +134,7 @@ public class FilesystemBackendModuleTest
String originalFilePath = filesBeforeMove.get(0).getAbsolutePath();
String movedFilePath = originalFilePath.replace(basePath, subPath);
filesystemBackendModule.moveFile(qInstance, table, originalFilePath, movedFilePath);
filesystemBackendModule.getActionBase().moveFile(qInstance, table, originalFilePath, movedFilePath);
List<File> filesAfterMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeMove.size() - 1, filesAfterMove.size(), "Should be one fewer file in the listing after moving one.");
@ -141,7 +142,7 @@ public class FilesystemBackendModuleTest
//////////////////////////////////////////////////////////////////////////
// move the file back and assert that the count goes back to the before //
//////////////////////////////////////////////////////////////////////////
filesystemBackendModule.moveFile(qInstance, table, movedFilePath, originalFilePath);
filesystemBackendModule.getActionBase().moveFile(qInstance, table, movedFilePath, originalFilePath);
List<File> filesAfterMoveBack = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeMove.size(), filesAfterMoveBack.size(), "Should be original number of files after moving back");
@ -166,7 +167,7 @@ public class FilesystemBackendModuleTest
Assertions.assertThrows(FilesystemException.class, () ->
{
FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
filesystemBackendModule.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, movedFilePath);
filesystemBackendModule.getActionBase().moveFile(qInstance, table, PATH_THAT_WONT_EXIST, movedFilePath);
});
}

33
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java
View File

@ -29,6 +29,8 @@ import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import static org.junit.jupiter.api.Assertions.fail;
@ -40,6 +42,28 @@ import static org.junit.jupiter.api.Assertions.fail;
public class FilesystemActionTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
public void beforeEach() throws Exception
{
primeFilesystem();
}
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
public void afterEach() throws Exception
{
cleanFilesystem();
}
/*******************************************************************************
** Set up the file system
*******************************************************************************/
@ -61,14 +85,17 @@ public class FilesystemActionTest
/*******************************************************************************
** Write some data files into the directory for the filesystem module.
*******************************************************************************/
private void writePersonFiles(File baseDirectory) throws IOException
{
String fullPath = baseDirectory.getAbsolutePath();
if (TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
if (TestUtils.defineLocalFilesystemJSONPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
{
if (StringUtils.hasContent(details.getPath()))
if (StringUtils.hasContent(details.getBasePath()))
{
fullPath += File.separatorChar + details.getPath();
fullPath += File.separatorChar + details.getBasePath();
}
}
fullPath += File.separatorChar;

26
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java
View File

@ -28,9 +28,7 @@ import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@ -40,28 +38,6 @@ import org.junit.jupiter.api.Test;
public class FilesystemQueryActionTest extends FilesystemActionTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
public void beforeEach() throws Exception
{
super.primeFilesystem();
}
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
public void afterEach() throws Exception
{
super.cleanFilesystem();
}
/*******************************************************************************
**
*******************************************************************************/
@ -85,7 +61,7 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
{
QueryRequest queryRequest = new QueryRequest();
queryRequest.setInstance(TestUtils.defineInstance());
queryRequest.setTableName(TestUtils.defineLocalFilesystemCSVPersonTable().getName());
queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
return queryRequest;
}

2
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java
View File

@ -71,7 +71,7 @@ class FilesystemBackendMetaDataTest
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
assertThat(deserialized).usingRecursiveComparison()
.ignoringFields("hasBeenValidated")
.ignoringFields("hasBeenValidated") // note, this field is @JsonIgnore
.isEqualTo(qInstance);
}
}

141
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java
View File

@ -23,20 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et
import java.io.File;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
import com.kingsrook.qqq.backend.core.callbacks.NoopCallback;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.apache.commons.io.FileUtils;
@ -56,17 +55,11 @@ public class BasicETLCleanupSourceFilesFunctionTest
**
*******************************************************************************/
@Test
public void testDelete() throws Exception
public void testDelete1Record1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
RunFunctionResult runFunctionResult = runFunction(qInstance, filePath, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, "delete",
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, "/tmp/trash"));
assertNull(runFunctionResult.getError());
assertFalse(new File(filePath).exists(), "File should have been deleted.");
testDelete(qInstance, List.of(filePath));
}
@ -75,21 +68,11 @@ public class BasicETLCleanupSourceFilesFunctionTest
**
*******************************************************************************/
@Test
public void testMove() throws Exception
public void testDelete2Records1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
String trashDir = File.separator + "tmp" + File.separator + "trash";
RunFunctionResult runFunctionResult = runFunction(qInstance, filePath, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, "move",
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir));
assertNull(runFunctionResult.getError());
assertFalse(new File(filePath).exists(), "File should have been moved.");
String movedPath = trashDir + File.separator + (new File(filePath).getName());
assertTrue(new File(movedPath).exists(), "File should have been moved.");
testDelete(qInstance, List.of(filePath, filePath));
}
@ -97,25 +80,127 @@ public class BasicETLCleanupSourceFilesFunctionTest
/*******************************************************************************
**
*******************************************************************************/
private RunFunctionResult runFunction(QInstance qInstance, String filePath, Map<String, String> values) throws Exception
@Test
public void testDelete2Record2File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath1 = getRandomFilePathPersonTable(qInstance);
String filePath2 = getRandomFilePathPersonTable(qInstance);
testDelete(qInstance, List.of(filePath1, filePath2));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMove1Record1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
testMove(qInstance, List.of(filePath));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMove2Records1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
testMove(qInstance, List.of(filePath, filePath));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMove2Record2File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath1 = getRandomFilePathPersonTable(qInstance);
String filePath2 = getRandomFilePathPersonTable(qInstance);
testMove(qInstance, List.of(filePath1, filePath2));
}
/*******************************************************************************
**
*******************************************************************************/
private void testDelete(QInstance qInstance, List<String> filePaths) throws Exception
{
RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_DELETE,
// todo - even though this field isn't needed, since we gave a value of "delete"
// the RunFunctionAction considers any missing input to be an error...
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, ""));
assertNull(runFunctionResult.getError());
for(String filePath : filePaths)
{
assertFalse(new File(filePath).exists(), "File should have been deleted.");
}
}
/*******************************************************************************
**
*******************************************************************************/
private void testMove(QInstance qInstance, List<String> filePaths) throws Exception
{
String trashDir = File.separator + "tmp" + File.separator + "trash";
RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_MOVE,
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir));
assertNull(runFunctionResult.getError());
for(String filePath : filePaths)
{
assertFalse(new File(filePath).exists(), "File should have been moved.");
String movedPath = trashDir + File.separator + (new File(filePath).getName());
assertTrue(new File(movedPath).exists(), "File should have been moved.");
}
}
/*******************************************************************************
**
*******************************************************************************/
private RunFunctionResult runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception
{
QFunctionMetaData qFunctionMetaData = new BasicETLCleanupSourceFilesFunction().defineFunctionMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addFunction(qFunctionMetaData);
qInstance.addProcess(qProcessMetaData);
HashSet<String> filePathsSet = new HashSet<>(filePaths);
for(String filePath : filePathsSet)
{
File file = new File(filePath);
FileUtils.writeStringToFile(file, "content");
}
List<QRecord> records = List.of(new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath));
// List<QRecord> records = filePaths.stream()
// .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList();
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(qFunctionMetaData.getName());
runFunctionRequest.setProcessName(qProcessMetaData.getName());
runFunctionRequest.setCallback(new NoopCallback());
runFunctionRequest.setRecords(records);
// runFunctionRequest.setRecords(records);
runFunctionRequest.setSession(TestUtils.getMockSession());
runFunctionRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
runFunctionRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
runFunctionRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
for(Map.Entry<String, String> entry : values.entrySet())
{
@ -135,7 +220,7 @@ public class BasicETLCleanupSourceFilesFunctionTest
{
FilesystemBackendMetaData backend = (FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS).getBackendDetails();
String tablePath = backend.getBasePath() + File.separator + backendDetails.getPath();
String tablePath = backend.getBasePath() + File.separator + backendDetails.getBasePath();
String filePath = tablePath + File.separator + UUID.randomUUID();
return filePath;
}

266
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java Normal file
View File

@ -0,0 +1,266 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync;
import java.util.List;
import java.util.stream.Collectors;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModuleSubclassForTest;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for FilesystemSyncProcess using S3 backend
*******************************************************************************/
class FilesystemSyncProcessS3Test extends BaseS3Test
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void test() throws Exception
{
QBackendModuleDispatcher.registerBackendModule(new S3BackendModuleSubclassForTest());
QInstance qInstance = TestUtils.defineInstance();
String sourceBucket = "source-bucket";
String archiveBucket = "archive-bucket";
String processingBucket = "processing-bucket";
getAmazonS3().createBucket(sourceBucket);
getAmazonS3().createBucket(archiveBucket);
getAmazonS3().createBucket(processingBucket);
S3BackendMetaData sourceBackend = defineBackend(qInstance, "source", sourceBucket);
S3BackendMetaData archiveBackend = defineBackend(qInstance, "archive", archiveBucket);
S3BackendMetaData processingBackend = defineBackend(qInstance, "processing", processingBucket);
QTableMetaData sourceTable = defineTable(qInstance, "source", sourceBackend, "source", "*/l3/*.csv");
QTableMetaData archiveTable = defineTable(qInstance, "archive", archiveBackend, "archive", "*/l3/*.csv");
QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
qInstance.addProcess(process);
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
///////////////////////////
// write some test files //
///////////////////////////
writeTestFile(sourceBackend, sourceTable, "foo/l3/1.csv", "x");
writeTestFile(sourceBackend, sourceTable, "bar/l3/2.csv", "x");
writeTestFile(archiveBackend, archiveTable, "foo/l3/1.csv", "x");
printTableListing(sourceBackend, sourceTable);
printTableListing(archiveBackend, archiveTable);
printTableListing(processingBackend, processingTable);
//////////////////////
// run the function //
//////////////////////
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(function.getName());
runFunctionRequest.setProcessName(process.getName());
runFunctionRequest.setSession(TestUtils.getMockSession());
RunFunctionAction runFunctionAction = new RunFunctionAction();
RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
System.out.println(runFunctionResult);
printTableListing(sourceBackend, sourceTable);
printTableListing(archiveBackend, archiveTable);
printTableListing(processingBackend, processingTable);
assertTableListing(archiveBackend, archiveTable, "root/archive/foo/l3/1.csv", "root/archive/bar/l3/2.csv");
assertTableListing(processingBackend, processingTable, "root/processing/bar/l3/2.csv");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testArchiveAndProcessingInSameBucket() throws Exception
{
QBackendModuleDispatcher.registerBackendModule(new S3BackendModuleSubclassForTest());
QInstance qInstance = TestUtils.defineInstance();
String vendorBucket = "vendor-bucket";
String localBucket = "local-bucket";
getAmazonS3().createBucket(vendorBucket);
getAmazonS3().createBucket(localBucket);
S3BackendMetaData vendorBackend = defineBackend(qInstance, "source", vendorBucket);
S3BackendMetaData localBackend = defineBackend(qInstance, "archive", localBucket);
QTableMetaData sourceTable = defineTable(qInstance, "source", vendorBackend, "source", "*/l3/*.csv");
QTableMetaData archiveTable = defineTable(qInstance, "archive", localBackend, "archive", "*/l3/*.csv");
QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
qInstance.addProcess(process);
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
///////////////////////////
// write some test files //
///////////////////////////
writeTestFile(vendorBackend, sourceTable, "foo/l3/1.csv", "x");
writeTestFile(vendorBackend, sourceTable, "bar/l3/2.csv", "x");
writeTestFile(localBackend, archiveTable, "foo/l3/1.csv", "x");
printTableListing(vendorBackend, sourceTable);
printTableListing(localBackend, archiveTable);
printTableListing(localBackend, processingTable);
//////////////////////
// run the function //
//////////////////////
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(function.getName());
runFunctionRequest.setProcessName(process.getName());
runFunctionRequest.setSession(TestUtils.getMockSession());
RunFunctionAction runFunctionAction = new RunFunctionAction();
RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
System.out.println(runFunctionResult);
printTableListing(vendorBackend, sourceTable);
printTableListing(localBackend, archiveTable);
printTableListing(localBackend, processingTable);
assertTableListing(localBackend, archiveTable, "root/archive/foo/l3/1.csv", "root/archive/bar/l3/2.csv");
assertTableListing(localBackend, processingTable, "root/processing/bar/l3/2.csv");
}
/*******************************************************************************
**
*******************************************************************************/
private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException
{
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
List<S3ObjectSummary> s3ObjectSummaries = actionBase.listFiles(table, backend);
assertEquals(paths.length, s3ObjectSummaries.size(), "Expected number of files in table: " + table.getName());
for(String path : paths)
{
assertTrue(s3ObjectSummaries.stream().anyMatch(s3o -> s3o.getKey().equals(path)),
"Path [" + path + "] should be in the listing, but was not. Full listing is: " +
s3ObjectSummaries.stream().map(S3ObjectSummary::getKey).collect(Collectors.joining(",")));
}
}
/*******************************************************************************
**
*******************************************************************************/
private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException
{
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
System.out.println("Files in: " + table.getName());
actionBase.listFiles(table, backend).forEach(o -> System.out.println(o.getKey()));
}
/*******************************************************************************
**
*******************************************************************************/
private void writeTestFile(S3BackendMetaData backend, QTableMetaData table, String name, String content) throws Exception
{
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
String fullPath = actionBase.getFullBasePath(table, backend);
actionBase.writeFile(backend, fullPath + "/" + name, content.getBytes());
}
/*******************************************************************************
**
*******************************************************************************/
private S3BackendMetaData defineBackend(QInstance qInstance, String which, String bucketName)
{
QBackendMetaData backendMetaData = new S3BackendMetaData()
.withBucketName(bucketName)
.withBasePath("root")
.withName("backend-" + which);
qInstance.addBackend(backendMetaData);
return (S3BackendMetaData) backendMetaData;
}
/*******************************************************************************
**
*******************************************************************************/
private QTableMetaData defineTable(QInstance qInstance, String which, QBackendMetaData backend, String path, String glob)
{
QTableMetaData qTableMetaData = new QTableMetaData()
.withName("table-" + which)
.withBackendName(backend.getName())
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withBackendDetails(new S3TableBackendDetails()
.withBasePath(path)
.withGlob(glob));
qInstance.addTable(qTableMetaData);
return (qTableMetaData);
}
}

119
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java Normal file
View File

@ -0,0 +1,119 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync;
import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for FilesystemSyncProcess
*******************************************************************************/
class FilesystemSyncProcessTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void test() throws Exception
{
QTableMetaData sourceTable = defineTable("source");
QTableMetaData archiveTable = defineTable("archive");
QTableMetaData processingTable = defineTable("processing");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
QInstance qInstance = TestUtils.defineInstance();
qInstance.addTable(sourceTable);
qInstance.addTable(archiveTable);
qInstance.addTable(processingTable);
qInstance.addProcess(process);
///////////////////////////
// write some test files //
///////////////////////////
String basePath = ((FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS)).getBasePath();
writeTestFile(basePath, sourceTable, "1.txt", "x");
writeTestFile(basePath, sourceTable, "2.txt", "x");
writeTestFile(basePath, archiveTable, "2.txt", "x");
//////////////////////
// run the function //
//////////////////////
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(function.getName());
runFunctionRequest.setProcessName(process.getName());
runFunctionRequest.setSession(TestUtils.getMockSession());
RunFunctionAction runFunctionAction = new RunFunctionAction();
RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
System.out.println(runFunctionResult);
}
/*******************************************************************************
**
*******************************************************************************/
private void writeTestFile(String basePath, QTableMetaData table, String name, String content) throws IOException
{
String path = ((FilesystemTableBackendDetails) table.getBackendDetails()).getBasePath();
File file = new File(basePath + "/" + path + "/" + name);
FileUtils.writeStringToFile(file, content);
}
/*******************************************************************************
**
*******************************************************************************/
private QTableMetaData defineTable(String subPath)
{
return new QTableMetaData()
.withName("table-" + subPath)
.withBackendName(TestUtils.BACKEND_NAME_LOCAL_FS)
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath(subPath));
}
}

1
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java
View File

@ -59,6 +59,7 @@ public class BaseS3Test
amazonS3.putObject(BUCKET_NAME, "0.csv", getCSVHeader());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/1.csv", getCSVData1());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
}

53
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleSubclassForTest.java Normal file
View File

@ -0,0 +1,53 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import cloud.localstack.awssdkv1.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
/*******************************************************************************
** Subclass of the S3Backend module, meant for use in unit tests, if/where we
** need to make sure we use the localstack version of the S3 client.
*******************************************************************************/
public class S3BackendModuleSubclassForTest extends S3BackendModule
{
/*******************************************************************************
** Seed the AbstractS3Action with an s3Utils object that has the localstack
** s3 client in it
*******************************************************************************/
@Override
public AbstractBaseFilesystemAction<S3ObjectSummary> getActionBase()
{
AbstractS3Action actionBase = (AbstractS3Action) super.getActionBase();
S3Utils s3Utils = new S3Utils();
s3Utils.setAmazonS3(TestUtils.getClientS3());
actionBase.setS3Utils(s3Utils);
return (actionBase);
}
}

41
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java
View File

@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@ -54,13 +55,14 @@ public class S3BackendModuleTest extends BaseS3Test
/////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then delete one, then re-list, and assert that we have one fewer //
/////////////////////////////////////////////////////////////////////////////////////////////
List<S3ObjectSummary> s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
List<S3ObjectSummary> s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
S3BackendModule s3BackendModule = new S3BackendModule();
s3BackendModule.setS3Utils(getS3Utils());
s3BackendModule.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey());
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
actionBase.setS3Utils(getS3Utils());
actionBase.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey());
List<S3ObjectSummary> s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
List<S3ObjectSummary> s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(),
"Should be one fewer file listed after deleting one.");
}
@ -80,13 +82,14 @@ public class S3BackendModuleTest extends BaseS3Test
// first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
// note, we'd like to detect the non-delete, but there's no such info back from aws it appears? //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<S3ObjectSummary> s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
List<S3ObjectSummary> s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
S3BackendModule s3BackendModule = new S3BackendModule();
s3BackendModule.setS3Utils(getS3Utils());
s3BackendModule.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST);
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
actionBase.setS3Utils(getS3Utils());
actionBase.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST);
List<S3ObjectSummary> s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
List<S3ObjectSummary> s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size(), s3ObjectSummariesAfterDelete.size(),
"Should be same number of files after deleting bogus path");
}
@ -107,18 +110,19 @@ public class S3BackendModuleTest extends BaseS3Test
// first list the files (non-recursively) - then move one into a sub-folder, then re-list, and //
// assert that we have one fewer then list again including sub-folders, and see the changed count //
////////////////////////////////////////////////////////////////////////////////////////////////////
List<S3ObjectSummary> s3ObjectSummariesBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
List<S3ObjectSummary> s3ObjectSummariesInSubFolderBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, subPath, false);
List<S3ObjectSummary> s3ObjectSummariesRecursiveBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, true);
List<S3ObjectSummary> s3ObjectSummariesBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
List<S3ObjectSummary> s3ObjectSummariesInSubFolderBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, subPath, "");
List<S3ObjectSummary> s3ObjectSummariesRecursiveBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**");
S3BackendModule s3BackendModule = new S3BackendModule();
s3BackendModule.setS3Utils(getS3Utils());
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
actionBase.setS3Utils(getS3Utils());
String key = s3ObjectSummariesBeforeMove.get(0).getKey();
s3BackendModule.moveFile(qInstance, table, key, key.replaceFirst(TEST_FOLDER, subPath));
actionBase.moveFile(qInstance, table, key, key.replaceFirst(TEST_FOLDER, subPath));
List<S3ObjectSummary> s3ObjectSummariesAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
List<S3ObjectSummary> s3ObjectSummariesRecursiveAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, true);
List<S3ObjectSummary> s3ObjectSummariesInSubFolderAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, subPath, false);
List<S3ObjectSummary> s3ObjectSummariesAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
List<S3ObjectSummary> s3ObjectSummariesRecursiveAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**");
List<S3ObjectSummary> s3ObjectSummariesInSubFolderAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, subPath, "");
Assertions.assertEquals(s3ObjectSummariesBeforeMove.size() - 1, s3ObjectSummariesAfterMove.size(),
"Should be one fewer file in the non-recursive listing after moving one.");
@ -141,10 +145,11 @@ public class S3BackendModuleTest extends BaseS3Test
String subPath = TEST_FOLDER + "/" + SUB_FOLDER;
S3BackendModule s3BackendModule = new S3BackendModule();
s3BackendModule.setS3Utils(getS3Utils());
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
actionBase.setS3Utils(getS3Utils());
Assertions.assertThrows(FilesystemException.class, () ->
s3BackendModule.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, subPath + "/" + UUID.randomUUID())
actionBase.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, subPath + "/" + UUID.randomUUID())
);
}

4
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java
View File

@ -52,7 +52,7 @@ class S3BackendMetaDataTest
System.out.println(JsonUtils.prettyPrint(json));
System.out.println(json);
String expectToContain = """
{"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","backendType":"s3","name":"s3"}""";
{"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","secretKey":null,"accessKey":null,"backendType":"s3","name":"s3","region":null}""";
assertTrue(json.contains(expectToContain));
}
@ -71,7 +71,7 @@ class S3BackendMetaDataTest
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
assertThat(deserialized).usingRecursiveComparison()
.ignoringFields("hasBeenValidated")
.ignoringFields("hasBeenValidated") // note, this field is @JsonIgnore
.isEqualTo(qInstance);
}
}

73
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/PathMatcherGlobTest.java Normal file
View File

@ -0,0 +1,73 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
import java.net.URI;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Verification for some of the behavior in the S3Utils - working with PathMatcher
** globs.
*******************************************************************************/
public class PathMatcherGlobTest
{
@Test
public void testPathMatcher() throws Exception
{
/////////////////////////////////////////////////////////////
// note: must start with for both the pattern and the uri //
/////////////////////////////////////////////////////////////
PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/*/acme/*/*.csv");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/1234.csv"))), "Glob should match");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/nj/acme/20220627/1234.csv"))), "Glob should match");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220628/1234.csv"))), "Glob should match");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/12345.csv"))), "Glob should match");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/beta/20220627/1234.csv"))), "Glob should not match (beta vs acme)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///something/stl/acme/20220627/1234.csv"))), "Glob should not match (wrong start path)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/csv"))), "Glob should not match (no file basename)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/1234.CSV"))), "Glob should not match (wrong case extension)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/extra/1234.csv"))), "Glob should not match (extra dir)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/extra/acme/20220627/1234.csv"))), "Glob should not match (extra dir)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/extra/stl/acme/20220627/1234.csv"))), "Glob should not match (extra dir)");
pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/**/acme/*/*.csv");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/extra/stl/acme/20220627/1234.csv"))), "Glob should match with extra dir");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/extra/extra2/stl/acme/20220627/1234.csv"))), "Glob should match with 2 extra dirs");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/acme/20220627/1234.csv"))), "Glob does not match with no dir for **");
pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/**");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/1234.csv"))), "Glob should match with extra dir");
pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/*");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///1234.csv"))), "Glob should match");
}
}

31
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import org.apache.commons.io.IOUtils;
@ -44,15 +45,23 @@ public class S3UtilsTest extends BaseS3Test
public void testListObjectsInBucketAtPath()
{
S3Utils s3Utils = getS3Utils();
assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/", false).size(), "Expected # of s3 objects without subfolders");
assertEquals(3, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/", true).size(), "Expected # of s3 objects with subfolders");
assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "/" + TEST_FOLDER + "/", false).size(), "With leading slash");
assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "/" + TEST_FOLDER, false).size(), "Without trailing slash");
assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "//" + TEST_FOLDER + "//", false).size(), "With multiple leading and trailing slashes");
assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, false).size(), "Just in the subfolder non-recursive");
assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, true).size(), "Just in the subfolder recursive");
assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "//" + SUB_FOLDER, true).size(), "Just in the subfolder recursive");
assertEquals(0, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "not-a-real-path/", true).size(), "In a non-existing folder");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders");
assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders");
assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders");
assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, "").size(), "Just in the subfolder non-recursive");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, "/**").size(), "Just in the subfolder recursive");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "//" + SUB_FOLDER, "/**").size(), "Just in the subfolder recursive, multi /");
assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "not-a-real-path/", "").size(), "In a non-existing folder");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string");
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
}
@ -64,11 +73,11 @@ public class S3UtilsTest extends BaseS3Test
public void testGetObjectAsInputStream() throws IOException
{
S3Utils s3Utils = getS3Utils();
S3ObjectSummary s3ObjectSummary = s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "test-files", true).get(0);
List<S3ObjectSummary> s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", "");
S3ObjectSummary s3ObjectSummary = s3ObjectSummaries.stream().filter(o -> o.getKey().contains("1.csv")).findAny().get();
InputStream inputStream = s3Utils.getObjectAsInputStream(s3ObjectSummary);
String csvFromS3 = IOUtils.toString(inputStream);
// todo - should check the filename somewhere, right?
assertEquals(getCSVData1(), csvFromS3, "File from S3 should match expected content");
}