QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

QQQ-14, QQQ-16 updated filesystem, s3 implementation, including passable version of basic ETL process & cleanup

Browse Source
This commit is contained in:
darin.kelkhoff
2022-06-23 14:53:06 -05:00
parent 62b015200a
commit 93a06cf1ab
17 changed files with 840 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java Normal file
View File

@ -0,0 +1,51 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
/*******************************************************************************
** Interface to add additional functionality commonly among the various filesystem
** module implementations.
*******************************************************************************/
public interface FilesystemBackendModuleInterface
{
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
*******************************************************************************/
void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException;
/*******************************************************************************
** Move a file from a source path, to a destination path.
**
** @throws FilesystemException if the move is known to have failed
*******************************************************************************/
void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException;
}

32
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemRecordBackendDetailFields.java Normal file
View File

@ -0,0 +1,32 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base;
/*******************************************************************************
** Define the field names (keys) to be used in the backendDetails structure
** under Records from this (these) modules.
*******************************************************************************/
public interface FilesystemRecordBackendDetailFields
{
String FULL_PATH = "fullPath";
}

6
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
View File

@ -59,6 +59,10 @@ public abstract class AbstractBaseFilesystemAction<FILE>
*******************************************************************************/
public abstract InputStream readFile(FILE file) throws IOException;
/*******************************************************************************
** Add backend details to records about the file that they are in.
*******************************************************************************/
protected abstract void addBackendDetailsToRecords(List<QRecord> recordsInFile, FILE file);
/*******************************************************************************
@ -133,6 +137,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
{
String fileContents = IOUtils.toString(readFile(file));
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
records.addAll(recordsInFile);
break;
@ -141,6 +146,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
{
String fileContents = IOUtils.toString(readFile(file));
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
records.addAll(recordsInFile);
break;

52
src/main/java/com/kingsrook/qqq/backend/module/filesystem/exceptions/FilesystemException.java Normal file
View File

@ -0,0 +1,52 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.exceptions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
/*******************************************************************************
** Checked exception to be thrown from actions within this module.
*******************************************************************************/
public class FilesystemException extends QException
{
/*******************************************************************************
** Constructor of message
**
*******************************************************************************/
public FilesystemException(String message)
{
super(message);
}
/*******************************************************************************
** Constructor of message & cause
**
*******************************************************************************/
public FilesystemException(String message, Throwable cause)
{
super(message, cause);
}
}

78
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java
View File

@ -22,26 +22,34 @@
package com.kingsrook.qqq.backend.module.filesystem.local;
import java.io.File;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemUpdateAction;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** QQQ Backend module for working with (local) Filesystems.
*******************************************************************************/
public class FilesystemBackendModule implements QBackendModuleInterface
public class FilesystemBackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface
{
private static final Logger LOG = LogManager.getLogger(FilesystemBackendModule.class);
/*******************************************************************************
@ -65,6 +73,7 @@ public class FilesystemBackendModule implements QBackendModuleInterface
}
/*******************************************************************************
** Method to identify the class used for table-backend details for this module.
*******************************************************************************/
@ -117,4 +126,71 @@ public class FilesystemBackendModule implements QBackendModuleInterface
{
return (new FilesystemDeleteAction());
}
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
*******************************************************************************/
@Override
public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
{
File file = new File(fileReference);
if(!file.exists())
{
//////////////////////////////////////////////////////////////////////////////////////////////
// if the file doesn't exist, just exit with noop. don't throw an error - that should only //
// happen if the "contract" of the method is broken, and the file still exists //
//////////////////////////////////////////////////////////////////////////////////////////////
LOG.debug("Not deleting file [{}], because it does not exist.", file);
return;
}
if(!file.delete())
{
throw (new FilesystemException("Failed to delete file: " + fileReference));
}
}
/*******************************************************************************
** Move a file from a source path, to a destination path.
**
** @throws FilesystemException if the delete is known to have failed
*******************************************************************************/
@Override
public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException
{
File sourceFile = new File(source);
File destinationFile = new File(destination);
File destinationParent = destinationFile.getParentFile();
if(!sourceFile.exists())
{
throw (new FilesystemException("Cannot move file " + source + ", as it does not exist."));
}
//////////////////////////////////////////////////////////////////////////////////////
// if the destination folder doesn't exist, try to make it - and fail if that fails //
//////////////////////////////////////////////////////////////////////////////////////
if(!destinationParent.exists())
{
LOG.debug("Making destination directory {} for move", destinationParent.getAbsolutePath());
if(!destinationParent.mkdirs())
{
throw (new FilesystemException("Failed to make destination directory " + destinationParent.getAbsolutePath() + " to move " + source + " into."));
}
}
if(!sourceFile.renameTo(destinationFile))
{
throw (new FilesystemException("Failed to move (rename) file " + source + " to " + destination));
}
}
}

26
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java
View File

@ -27,9 +27,12 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
@ -47,7 +50,14 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
{
String fullPath = getFullPath(table, backendBase);
File directory = new File(fullPath);
return Arrays.asList(directory.listFiles());
File[] files = directory.listFiles();
if(files == null)
{
return Collections.emptyList();
}
return (Arrays.stream(files).filter(File::isFile).toList());
}
@ -61,4 +71,18 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
return (new FileInputStream(file));
}
/*******************************************************************************
** Add backend details to records about the file that they are in.
*******************************************************************************/
@Override
protected void addBackendDetailsToRecords(List<QRecord> recordsInFile, File file)
{
recordsInFile.forEach(record ->
{
record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, file.getAbsolutePath());
});
}
}

110
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java Normal file
View File

@ -0,0 +1,110 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
import java.io.File;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
/*******************************************************************************
** Function body for performing the Cleanup step of a basic ETL process - e.g.,
** after the loading, delete or move the processed file(s).
*******************************************************************************/
public class BasicETLCleanupSourceFilesFunction implements FunctionBody
{
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
{
String sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
QTableMetaData table = runFunctionRequest.getInstance().getTable(sourceTableName);
QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(table.getBackendName());
if(!(module instanceof FilesystemBackendModuleInterface filesystemModule))
{
throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function."));
}
Set<String> sourceFiles = runFunctionRequest.getRecords().stream()
.map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))
.collect(Collectors.toSet());
for(String sourceFile : sourceFiles)
{
String moveOrDelete = runFunctionRequest.getValueString("moveOrDelete");
if("delete".equals(moveOrDelete))
{
filesystemModule.deleteFile(runFunctionRequest.getInstance(), table, sourceFile);
}
else if("move".equals(moveOrDelete))
{
String destinationForMoves = runFunctionRequest.getValueString("destinationForMoves");
if(!StringUtils.hasContent(destinationForMoves))
{
throw (new QException("Field [destinationForMoves] is missing a value."));
}
File file = new File(sourceFile);
String destinationPath = destinationForMoves + File.separator + file.getName();
filesystemModule.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath);
}
else
{
throw (new QException("Unexpected value [" + moveOrDelete + "] for field [moveOrDelete]. Must be either [move] or [delete]."));
}
}
}
/*******************************************************************************
**
*******************************************************************************/
public QFunctionMetaData defineFunctionMetaData()
{
return (new QFunctionMetaData()
.withName("cleanupSourceFiles")
.withCode(new QCodeReference()
.withName(this.getClass().getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.FUNCTION))
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData("moveOrDelete", QFieldType.STRING))
.addField(new QFieldMetaData("destinationForMoves", QFieldType.STRING))));
}
}

74
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java
View File

@ -23,25 +23,32 @@ package com.kingsrook.qqq.backend.module.filesystem.s3;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
/*******************************************************************************
** QQQ Backend module for working with AWS S3 filesystems
*******************************************************************************/
public class S3BackendModule implements QBackendModuleInterface
public class S3BackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface
{
private S3Utils s3Utils;
/*******************************************************************************
@ -72,7 +79,44 @@ public class S3BackendModule implements QBackendModuleInterface
@Override
public Class<? extends QTableBackendDetails> getTableBackendDetailsClass()
{
return S3TableBackendDetails.class;
return (S3TableBackendDetails.class);
}
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
*******************************************************************************/
@Override
public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
{
QBackendMetaData backend = instance.getBackend(table.getBackendName());
String bucketName = ((S3BackendMetaData) backend).getBucketName();
getS3Utils().deleteObject(bucketName, fileReference);
}
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @throws FilesystemException if the move is known to have failed
*******************************************************************************/
@Override
public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException
{
QBackendMetaData backend = instance.getBackend(table.getBackendName());
String bucketName = ((S3BackendMetaData) backend).getBucketName();
getS3Utils().moveObject(bucketName, source, destination);
}
@ -118,4 +162,30 @@ public class S3BackendModule implements QBackendModuleInterface
{
return (new S3DeleteAction());
}
/*******************************************************************************
** Setter for s3Utils
*******************************************************************************/
public void setS3Utils(S3Utils s3Utils)
{
this.s3Utils = s3Utils;
}
/*******************************************************************************
** Internal accessor for the s3Utils object - should always use this, not the field.
*******************************************************************************/
private S3Utils getS3Utils()
{
if(s3Utils == null)
{
s3Utils = new S3Utils();
}
return s3Utils;
}
}

17
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java
View File

@ -26,8 +26,10 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
@ -96,4 +98,19 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
{
return (getS3Utils().getObjectAsInputStream(s3ObjectSummary));
}
/*******************************************************************************
** Add backend details to records about the file that they are in.
*******************************************************************************/
@Override
protected void addBackendDetailsToRecords(List<QRecord> recordsInFile, S3ObjectSummary s3ObjectSummary)
{
recordsInFile.forEach(record ->
{
record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, s3ObjectSummary.getKey());
});
}
}

51
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java
View File

@ -31,12 +31,17 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Utility methods for working with AWS S3.
**
** Note: May need a constructor (or similar) in the future that takes the
** S3BackendMetaData - e.g., if we need some metaData to construct the AmazonS3
** (api client) object, such as region, or authentication.
*******************************************************************************/
public class S3Utils
{
@ -111,8 +116,9 @@ public class S3Utils
}
/*******************************************************************************
**
** Get the contents (as an InputStream) for an object in s3
*******************************************************************************/
public InputStream getObjectAsInputStream(S3ObjectSummary s3ObjectSummary)
{
@ -120,6 +126,48 @@ public class S3Utils
}
/*******************************************************************************
** Delete an object (file) from a bucket
*******************************************************************************/
public void deleteObject(String bucketName, String key) throws FilesystemException
{
//////////////////////////////////////////////////////////////////////////////////////////////
// note, aws s3 api does not appear to have any way to check the success or failure here... //
//////////////////////////////////////////////////////////////////////////////////////////////
try
{
getS3().deleteObject(bucketName, key);
}
catch(Exception e)
{
throw (new FilesystemException("Error deleting s3 object " + key + " in bucket " + bucketName, e));
}
}
/*******************************************************************************
** Move an object (file) within a bucket
*******************************************************************************/
public void moveObject(String bucketName, String source, String destination) throws FilesystemException
{
//////////////////////////////////////////////////////////////////////////////////////////////
// note, aws s3 api does not appear to have any way to check the success or failure here... //
//////////////////////////////////////////////////////////////////////////////////////////////
try
{
getS3().copyObject(bucketName, source, bucketName, destination);
getS3().deleteObject(bucketName, source);
}
catch(Exception e)
{
throw (new FilesystemException("Error moving s3 object " + source + " to " + destination + " in bucket " + bucketName, e));
}
}
/*******************************************************************************
** Setter for AmazonS3 client object.
*******************************************************************************/
@ -143,5 +191,4 @@ public class S3Utils
return s3;
}
}

15
src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -43,8 +43,15 @@ import org.apache.commons.io.FileUtils;
*******************************************************************************/
public class TestUtils
{
private static int testInstanceCounter = 0;
private final static String BASE_PATH = "/tmp/filesystem-tests";
public static final String TABLE_NAME_PERSON = "person";
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
///////////////////////////////////////////////////////////////////
// shouldn't be accessed directly, as we append a counter to it. //
///////////////////////////////////////////////////////////////////
public static final String BASE_PATH = "/tmp/filesystem-tests";
private static int testInstanceCounter = 0;
@ -126,7 +133,7 @@ public class TestUtils
public static QTableMetaData defineLocalFilesystemCSVPersonTable()
{
return new QTableMetaData()
.withName("person")
.withName(TABLE_NAME_PERSON)
.withLabel("Person")
.withBackendName(defineLocalFilesystemBackend().getName())
.withPrimaryKeyField("id")
@ -165,7 +172,7 @@ public class TestUtils
public static QTableMetaData defineS3CSVPersonTable()
{
return new QTableMetaData()
.withName("person-s3")
.withName(TABLE_NAME_PERSON_S3)
.withLabel("Person S3 Table")
.withBackendName(defineS3Backend().getName())
.withPrimaryKeyField("id")

174
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java Normal file
View File

@ -0,0 +1,174 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local;
import java.io.File;
import java.io.IOException;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for FilesystemBackendModule
*******************************************************************************/
public class FilesystemBackendModuleTest
{
private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist";
@BeforeEach
public void beforeEach() throws IOException
{
new FilesystemActionTest().primeFilesystem();
}
@AfterEach
public void afterEach() throws Exception
{
new FilesystemActionTest().cleanFilesystem();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDeleteFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
/////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then delete one, then re-list, and assert that we have one fewer //
/////////////////////////////////////////////////////////////////////////////////////////////
List<File> filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
filesystemBackendModule.deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath());
List<File> filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeDelete.size() - 1, filesAfterDelete.size(),
"Should be one fewer file listed after deleting one.");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDeleteFileDoesNotExist() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
// note, we'd like to detect the non-delete, but there's no such info back from aws it appears? //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<File> filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
filesystemBackendModule.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST);
List<File> filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeDelete.size(), filesAfterDelete.size(),
"Should be same number of files after deleting bogus path");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMoveFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
String subPath = basePath + File.separator + "subdir";
////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files (non-recursively) - then move one into a sub-folder, then re-list, and //
// assert that we have one fewer then list again including sub-folders, and see the changed count //
////////////////////////////////////////////////////////////////////////////////////////////////////
List<File> filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
String originalFilePath = filesBeforeMove.get(0).getAbsolutePath();
String movedFilePath = originalFilePath.replace(basePath, subPath);
filesystemBackendModule.moveFile(qInstance, table, originalFilePath, movedFilePath);
List<File> filesAfterMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeMove.size() - 1, filesAfterMove.size(), "Should be one fewer file in the listing after moving one.");
//////////////////////////////////////////////////////////////////////////
// move the file back and assert that the count goes back to the before //
//////////////////////////////////////////////////////////////////////////
filesystemBackendModule.moveFile(qInstance, table, movedFilePath, originalFilePath);
List<File> filesAfterMoveBack = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeMove.size(), filesAfterMoveBack.size(), "Should be original number of files after moving back");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMoveFileDoesNotExit() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
String subPath = basePath + File.separator + "subdir";
List<File> filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
String originalFilePath = filesBeforeMove.get(0).getAbsolutePath();
String movedFilePath = originalFilePath.replace(basePath, subPath);
Assertions.assertThrows(FilesystemException.class, () ->
{
FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
filesystemBackendModule.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, movedFilePath);
});
}
}

4
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java
View File

@ -43,7 +43,7 @@ public class FilesystemActionTest
/*******************************************************************************
** Set up the file system
*******************************************************************************/
protected void primeFilesystem() throws IOException
public void primeFilesystem() throws IOException
{
TestUtils.cleanInstanceFiles();
TestUtils.increaseTestInstanceCounter();
@ -94,7 +94,7 @@ public class FilesystemActionTest
/*******************************************************************************
**
*******************************************************************************/
protected void cleanFilesystem() throws IOException
public void cleanFilesystem() throws IOException
{
TestUtils.cleanInstanceFiles();
}

4
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java
View File

@ -27,6 +27,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
@ -70,6 +71,9 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
QueryRequest queryRequest = initQueryRequest();
QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows");
Assertions.assertTrue(queryResult.getRecords().stream()
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
"All records should have a full-path in their backend details, matching the test folder name");
}

2
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java
View File

@ -35,7 +35,7 @@ import org.junit.jupiter.api.extension.ExtendWith;
/*******************************************************************************
**
** Base class for tests that want to be able to work with localstack s3.
*******************************************************************************/
@ExtendWith(LocalstackDockerExtension.class)
@LocalstackDockerProperties(services = { ServiceName.S3 }, portEdge = "2960", portElasticSearch = "2961")

151
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java Normal file
View File

@ -0,0 +1,151 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3;
import java.util.List;
import java.util.UUID;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for S3BackendModule
*******************************************************************************/
public class S3BackendModuleTest extends BaseS3Test
{
private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist";
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDeleteFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
/////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then delete one, then re-list, and assert that we have one fewer //
/////////////////////////////////////////////////////////////////////////////////////////////
List<S3ObjectSummary> s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
S3BackendModule s3BackendModule = new S3BackendModule();
s3BackendModule.setS3Utils(getS3Utils());
s3BackendModule.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey());
List<S3ObjectSummary> s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(),
"Should be one fewer file listed after deleting one.");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDeleteFileDoesNotExist() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
// note, we'd like to detect the non-delete, but there's no such info back from aws it appears? //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<S3ObjectSummary> s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
S3BackendModule s3BackendModule = new S3BackendModule();
s3BackendModule.setS3Utils(getS3Utils());
s3BackendModule.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST);
List<S3ObjectSummary> s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size(), s3ObjectSummariesAfterDelete.size(),
"Should be same number of files after deleting bogus path");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMoveFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
String subPath = TEST_FOLDER + "/" + SUB_FOLDER;
////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files (non-recursively) - then move one into a sub-folder, then re-list, and //
// assert that we have one fewer then list again including sub-folders, and see the changed count //
////////////////////////////////////////////////////////////////////////////////////////////////////
List<S3ObjectSummary> s3ObjectSummariesBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
List<S3ObjectSummary> s3ObjectSummariesInSubFolderBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, subPath, false);
List<S3ObjectSummary> s3ObjectSummariesRecursiveBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, true);
S3BackendModule s3BackendModule = new S3BackendModule();
s3BackendModule.setS3Utils(getS3Utils());
String key = s3ObjectSummariesBeforeMove.get(0).getKey();
s3BackendModule.moveFile(qInstance, table, key, key.replaceFirst(TEST_FOLDER, subPath));
List<S3ObjectSummary> s3ObjectSummariesAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false);
List<S3ObjectSummary> s3ObjectSummariesRecursiveAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, true);
List<S3ObjectSummary> s3ObjectSummariesInSubFolderAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, subPath, false);
Assertions.assertEquals(s3ObjectSummariesBeforeMove.size() - 1, s3ObjectSummariesAfterMove.size(),
"Should be one fewer file in the non-recursive listing after moving one.");
Assertions.assertEquals(s3ObjectSummariesRecursiveBeforeMove.size(), s3ObjectSummariesRecursiveAfterMove.size(),
"Should be same number of files in the recursive listing before and after the move");
Assertions.assertEquals(s3ObjectSummariesInSubFolderBeforeMove.size() + 1, s3ObjectSummariesInSubFolderAfterMove.size(),
"Should be one move file in the sub-folder listing after moving one.");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMoveFileDoesNotExit() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
String subPath = TEST_FOLDER + "/" + SUB_FOLDER;
S3BackendModule s3BackendModule = new S3BackendModule();
s3BackendModule.setS3Utils(getS3Utils());
Assertions.assertThrows(FilesystemException.class, () ->
s3BackendModule.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, subPath + "/" + UUID.randomUUID())
);
}
}

8
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java
View File

@ -27,6 +27,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@ -44,11 +45,14 @@ public class S3QueryActionTest extends BaseS3Test
@Test
public void testQuery1() throws QException
{
QueryRequest queryRequest = initQueryRequest();
QueryRequest queryRequest = initQueryRequest();
S3QueryAction s3QueryAction = new S3QueryAction();
s3QueryAction.setS3Utils(getS3Utils());
QueryResult queryResult = s3QueryAction.execute(queryRequest);
QueryResult queryResult = s3QueryAction.execute(queryRequest);
Assertions.assertEquals(5, queryResult.getRecords().size(), "Expected # of rows from unfiltered query");
Assertions.assertTrue(queryResult.getRecords().stream()
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseS3Test.TEST_FOLDER)),
"All records should have a full-path in their backend details, matching the test folder name");
}