diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java new file mode 100644 index 00000000..9a7578a4 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java @@ -0,0 +1,51 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base; + + +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; + + +/******************************************************************************* + ** Interface to add additional functionality commonly among the various filesystem + ** module implementations. + *******************************************************************************/ +public interface FilesystemBackendModuleInterface +{ + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + *******************************************************************************/ + void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException; + + /******************************************************************************* + ** Move a file from a source path, to a destination path. + ** + ** @throws FilesystemException if the move is known to have failed + *******************************************************************************/ + void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException; +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemRecordBackendDetailFields.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemRecordBackendDetailFields.java new file mode 100644 index 00000000..415aa06f --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemRecordBackendDetailFields.java @@ -0,0 +1,32 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base; + + +/******************************************************************************* + ** Define the field names (keys) to be used in the backendDetails structure + ** under Records from this (these) modules. + *******************************************************************************/ +public interface FilesystemRecordBackendDetailFields +{ + String FULL_PATH = "fullPath"; +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 07b5f3dc..b8e07308 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -59,6 +59,10 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ public abstract InputStream readFile(FILE file) throws IOException; + /******************************************************************************* + ** Add backend details to records about the file that they are in. + *******************************************************************************/ + protected abstract void addBackendDetailsToRecords(List recordsInFile, FILE file); /******************************************************************************* @@ -133,6 +137,7 @@ public abstract class AbstractBaseFilesystemAction { String fileContents = IOUtils.toString(readFile(file)); List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); records.addAll(recordsInFile); break; @@ -141,6 +146,7 @@ public abstract class AbstractBaseFilesystemAction { String fileContents = IOUtils.toString(readFile(file)); List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); records.addAll(recordsInFile); break; diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/exceptions/FilesystemException.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/exceptions/FilesystemException.java new file mode 100644 index 00000000..cbda757b --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/exceptions/FilesystemException.java @@ -0,0 +1,52 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.exceptions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; + + +/******************************************************************************* + ** Checked exception to be thrown from actions within this module. + *******************************************************************************/ +public class FilesystemException extends QException +{ + /******************************************************************************* + ** Constructor of message + ** + *******************************************************************************/ + public FilesystemException(String message) + { + super(message); + } + + + + /******************************************************************************* + ** Constructor of message & cause + ** + *******************************************************************************/ + public FilesystemException(String message, Throwable cause) + { + super(message, cause); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java index 3a9bca3e..09594b99 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java @@ -22,26 +22,34 @@ package com.kingsrook.qqq.backend.module.filesystem.local; +import java.io.File; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemUpdateAction; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* ** QQQ Backend module for working with (local) Filesystems. *******************************************************************************/ -public class FilesystemBackendModule implements QBackendModuleInterface +public class FilesystemBackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface { + private static final Logger LOG = LogManager.getLogger(FilesystemBackendModule.class); /******************************************************************************* @@ -65,6 +73,7 @@ public class FilesystemBackendModule implements QBackendModuleInterface } + /******************************************************************************* ** Method to identify the class used for table-backend details for this module. *******************************************************************************/ @@ -117,4 +126,71 @@ public class FilesystemBackendModule implements QBackendModuleInterface { return (new FilesystemDeleteAction()); } + + + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + *******************************************************************************/ + @Override + public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + { + File file = new File(fileReference); + if(!file.exists()) + { + ////////////////////////////////////////////////////////////////////////////////////////////// + // if the file doesn't exist, just exit with noop. don't throw an error - that should only // + // happen if the "contract" of the method is broken, and the file still exists // + ////////////////////////////////////////////////////////////////////////////////////////////// + LOG.debug("Not deleting file [{}], because it does not exist.", file); + return; + } + + if(!file.delete()) + { + throw (new FilesystemException("Failed to delete file: " + fileReference)); + } + } + + + + /******************************************************************************* + ** Move a file from a source path, to a destination path. + ** + ** @throws FilesystemException if the delete is known to have failed + *******************************************************************************/ + @Override + public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException + { + File sourceFile = new File(source); + File destinationFile = new File(destination); + File destinationParent = destinationFile.getParentFile(); + + if(!sourceFile.exists()) + { + throw (new FilesystemException("Cannot move file " + source + ", as it does not exist.")); + } + + ////////////////////////////////////////////////////////////////////////////////////// + // if the destination folder doesn't exist, try to make it - and fail if that fails // + ////////////////////////////////////////////////////////////////////////////////////// + if(!destinationParent.exists()) + { + LOG.debug("Making destination directory {} for move", destinationParent.getAbsolutePath()); + if(!destinationParent.mkdirs()) + { + throw (new FilesystemException("Failed to make destination directory " + destinationParent.getAbsolutePath() + " to move " + source + " into.")); + } + } + + if(!sourceFile.renameTo(destinationFile)) + { + throw (new FilesystemException("Failed to move (rename) file " + source + " to " + destination)); + } + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index fde5c1d5..91b5654b 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -27,9 +27,12 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; @@ -47,7 +50,14 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction { String fullPath = getFullPath(table, backendBase); File directory = new File(fullPath); - return Arrays.asList(directory.listFiles()); + File[] files = directory.listFiles(); + + if(files == null) + { + return Collections.emptyList(); + } + + return (Arrays.stream(files).filter(File::isFile).toList()); } @@ -61,4 +71,18 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction return (new FileInputStream(file)); } + + + /******************************************************************************* + ** Add backend details to records about the file that they are in. + *******************************************************************************/ + @Override + protected void addBackendDetailsToRecords(List recordsInFile, File file) + { + recordsInFile.forEach(record -> + { + record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, file.getAbsolutePath()); + }); + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java new file mode 100644 index 00000000..f9a6a26f --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java @@ -0,0 +1,110 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic; + + +import java.io.File; +import java.util.Set; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; + + +/******************************************************************************* + ** Function body for performing the Cleanup step of a basic ETL process - e.g., + ** after the loading, delete or move the processed file(s). + *******************************************************************************/ +public class BasicETLCleanupSourceFilesFunction implements FunctionBody +{ + @Override + public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + { + String sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE); + QTableMetaData table = runFunctionRequest.getInstance().getTable(sourceTableName); + QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(table.getBackendName()); + if(!(module instanceof FilesystemBackendModuleInterface filesystemModule)) + { + throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function.")); + } + + Set sourceFiles = runFunctionRequest.getRecords().stream() + .map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH)) + .collect(Collectors.toSet()); + for(String sourceFile : sourceFiles) + { + String moveOrDelete = runFunctionRequest.getValueString("moveOrDelete"); + if("delete".equals(moveOrDelete)) + { + filesystemModule.deleteFile(runFunctionRequest.getInstance(), table, sourceFile); + } + else if("move".equals(moveOrDelete)) + { + String destinationForMoves = runFunctionRequest.getValueString("destinationForMoves"); + if(!StringUtils.hasContent(destinationForMoves)) + { + throw (new QException("Field [destinationForMoves] is missing a value.")); + } + File file = new File(sourceFile); + String destinationPath = destinationForMoves + File.separator + file.getName(); + filesystemModule.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath); + } + else + { + throw (new QException("Unexpected value [" + moveOrDelete + "] for field [moveOrDelete]. Must be either [move] or [delete].")); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QFunctionMetaData defineFunctionMetaData() + { + return (new QFunctionMetaData() + .withName("cleanupSourceFiles") + .withCode(new QCodeReference() + .withName(this.getClass().getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.FUNCTION)) + .withInputData(new QFunctionInputMetaData() + .addField(new QFieldMetaData("moveOrDelete", QFieldType.STRING)) + .addField(new QFieldMetaData("destinationForMoves", QFieldType.STRING)))); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java index 7c574cf3..33bffb0b 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java @@ -23,25 +23,32 @@ package com.kingsrook.qqq.backend.module.filesystem.s3; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; /******************************************************************************* ** QQQ Backend module for working with AWS S3 filesystems *******************************************************************************/ -public class S3BackendModule implements QBackendModuleInterface +public class S3BackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface { + private S3Utils s3Utils; + /******************************************************************************* @@ -72,7 +79,44 @@ public class S3BackendModule implements QBackendModuleInterface @Override public Class getTableBackendDetailsClass() { - return S3TableBackendDetails.class; + return (S3TableBackendDetails.class); + } + + + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + *******************************************************************************/ + @Override + public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + { + QBackendMetaData backend = instance.getBackend(table.getBackendName()); + String bucketName = ((S3BackendMetaData) backend).getBucketName(); + + getS3Utils().deleteObject(bucketName, fileReference); + } + + + + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the move is known to have failed + *******************************************************************************/ + @Override + public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException + { + QBackendMetaData backend = instance.getBackend(table.getBackendName()); + String bucketName = ((S3BackendMetaData) backend).getBucketName(); + + getS3Utils().moveObject(bucketName, source, destination); } @@ -118,4 +162,30 @@ public class S3BackendModule implements QBackendModuleInterface { return (new S3DeleteAction()); } + + + + /******************************************************************************* + ** Setter for s3Utils + *******************************************************************************/ + public void setS3Utils(S3Utils s3Utils) + { + this.s3Utils = s3Utils; + } + + + + /******************************************************************************* + ** Internal accessor for the s3Utils object - should always use this, not the field. + *******************************************************************************/ + private S3Utils getS3Utils() + { + if(s3Utils == null) + { + s3Utils = new S3Utils(); + } + + return s3Utils; + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 12c69959..1fb11d34 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -26,8 +26,10 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; @@ -96,4 +98,19 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction recordsInFile, S3ObjectSummary s3ObjectSummary) + { + recordsInFile.forEach(record -> + { + record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, s3ObjectSummary.getKey()); + }); + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java index 4dfd4f1c..65a25367 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java @@ -31,12 +31,17 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.ListObjectsV2Request; import com.amazonaws.services.s3.model.ListObjectsV2Result; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; /******************************************************************************* ** Utility methods for working with AWS S3. + ** + ** Note: May need a constructor (or similar) in the future that takes the + ** S3BackendMetaData - e.g., if we need some metaData to construct the AmazonS3 + ** (api client) object, such as region, or authentication. *******************************************************************************/ public class S3Utils { @@ -111,8 +116,9 @@ public class S3Utils } + /******************************************************************************* - ** + ** Get the contents (as an InputStream) for an object in s3 *******************************************************************************/ public InputStream getObjectAsInputStream(S3ObjectSummary s3ObjectSummary) { @@ -120,6 +126,48 @@ public class S3Utils } + + /******************************************************************************* + ** Delete an object (file) from a bucket + *******************************************************************************/ + public void deleteObject(String bucketName, String key) throws FilesystemException + { + ////////////////////////////////////////////////////////////////////////////////////////////// + // note, aws s3 api does not appear to have any way to check the success or failure here... // + ////////////////////////////////////////////////////////////////////////////////////////////// + try + { + getS3().deleteObject(bucketName, key); + } + catch(Exception e) + { + throw (new FilesystemException("Error deleting s3 object " + key + " in bucket " + bucketName, e)); + } + } + + + + /******************************************************************************* + ** Move an object (file) within a bucket + *******************************************************************************/ + public void moveObject(String bucketName, String source, String destination) throws FilesystemException + { + ////////////////////////////////////////////////////////////////////////////////////////////// + // note, aws s3 api does not appear to have any way to check the success or failure here... // + ////////////////////////////////////////////////////////////////////////////////////////////// + try + { + getS3().copyObject(bucketName, source, bucketName, destination); + getS3().deleteObject(bucketName, source); + } + catch(Exception e) + { + throw (new FilesystemException("Error moving s3 object " + source + " to " + destination + " in bucket " + bucketName, e)); + } + } + + + /******************************************************************************* ** Setter for AmazonS3 client object. *******************************************************************************/ @@ -143,5 +191,4 @@ public class S3Utils return s3; } - } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index a1f7ae38..17e7f266 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -43,8 +43,15 @@ import org.apache.commons.io.FileUtils; *******************************************************************************/ public class TestUtils { - private static int testInstanceCounter = 0; - private final static String BASE_PATH = "/tmp/filesystem-tests"; + public static final String TABLE_NAME_PERSON = "person"; + public static final String TABLE_NAME_PERSON_S3 = "person-s3"; + + /////////////////////////////////////////////////////////////////// + // shouldn't be accessed directly, as we append a counter to it. // + /////////////////////////////////////////////////////////////////// + public static final String BASE_PATH = "/tmp/filesystem-tests"; + + private static int testInstanceCounter = 0; @@ -126,7 +133,7 @@ public class TestUtils public static QTableMetaData defineLocalFilesystemCSVPersonTable() { return new QTableMetaData() - .withName("person") + .withName(TABLE_NAME_PERSON) .withLabel("Person") .withBackendName(defineLocalFilesystemBackend().getName()) .withPrimaryKeyField("id") @@ -165,7 +172,7 @@ public class TestUtils public static QTableMetaData defineS3CSVPersonTable() { return new QTableMetaData() - .withName("person-s3") + .withName(TABLE_NAME_PERSON_S3) .withLabel("Person S3 Table") .withBackendName(defineS3Backend().getName()) .withPrimaryKeyField("id") diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java new file mode 100644 index 00000000..d6e1e681 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java @@ -0,0 +1,174 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local; + + +import java.io.File; +import java.io.IOException; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for FilesystemBackendModule + *******************************************************************************/ +public class FilesystemBackendModuleTest +{ + private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist"; + + + + @BeforeEach + public void beforeEach() throws IOException + { + new FilesystemActionTest().primeFilesystem(); + } + + + + @AfterEach + public void afterEach() throws Exception + { + new FilesystemActionTest().cleanFilesystem(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteFile() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // first list the files - then delete one, then re-list, and assert that we have one fewer // + ///////////////////////////////////////////////////////////////////////////////////////////// + List filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + + FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); + filesystemBackendModule.deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath()); + + List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + Assertions.assertEquals(filesBeforeDelete.size() - 1, filesAfterDelete.size(), + "Should be one fewer file listed after deleting one."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteFileDoesNotExist() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count // + // note, we'd like to detect the non-delete, but there's no such info back from aws it appears? // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + + FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); + filesystemBackendModule.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + + List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + Assertions.assertEquals(filesBeforeDelete.size(), filesAfterDelete.size(), + "Should be same number of files after deleting bogus path"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMoveFile() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON); + String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath(); + String subPath = basePath + File.separator + "subdir"; + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // first list the files (non-recursively) - then move one into a sub-folder, then re-list, and // + // assert that we have one fewer then list again including sub-folders, and see the changed count // + //////////////////////////////////////////////////////////////////////////////////////////////////// + List filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + + FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); + String originalFilePath = filesBeforeMove.get(0).getAbsolutePath(); + String movedFilePath = originalFilePath.replace(basePath, subPath); + + filesystemBackendModule.moveFile(qInstance, table, originalFilePath, movedFilePath); + + List filesAfterMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + Assertions.assertEquals(filesBeforeMove.size() - 1, filesAfterMove.size(), "Should be one fewer file in the listing after moving one."); + + ////////////////////////////////////////////////////////////////////////// + // move the file back and assert that the count goes back to the before // + ////////////////////////////////////////////////////////////////////////// + filesystemBackendModule.moveFile(qInstance, table, movedFilePath, originalFilePath); + + List filesAfterMoveBack = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + Assertions.assertEquals(filesBeforeMove.size(), filesAfterMoveBack.size(), "Should be original number of files after moving back"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMoveFileDoesNotExit() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON); + String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath(); + String subPath = basePath + File.separator + "subdir"; + List filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + String originalFilePath = filesBeforeMove.get(0).getAbsolutePath(); + String movedFilePath = originalFilePath.replace(basePath, subPath); + + Assertions.assertThrows(FilesystemException.class, () -> + { + FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); + filesystemBackendModule.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, movedFilePath); + }); + + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java index 39ebf12c..9e9b38a2 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java @@ -43,7 +43,7 @@ public class FilesystemActionTest /******************************************************************************* ** Set up the file system *******************************************************************************/ - protected void primeFilesystem() throws IOException + public void primeFilesystem() throws IOException { TestUtils.cleanInstanceFiles(); TestUtils.increaseTestInstanceCounter(); @@ -94,7 +94,7 @@ public class FilesystemActionTest /******************************************************************************* ** *******************************************************************************/ - protected void cleanFilesystem() throws IOException + public void cleanFilesystem() throws IOException { TestUtils.cleanInstanceFiles(); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java index 80287082..f624cb5a 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java @@ -27,6 +27,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -70,6 +71,9 @@ public class FilesystemQueryActionTest extends FilesystemActionTest QueryRequest queryRequest = initQueryRequest(); QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest); Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows"); + Assertions.assertTrue(queryResult.getRecords().stream() + .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)), + "All records should have a full-path in their backend details, matching the test folder name"); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java index aef56d11..3563d91c 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java @@ -35,7 +35,7 @@ import org.junit.jupiter.api.extension.ExtendWith; /******************************************************************************* - ** + ** Base class for tests that want to be able to work with localstack s3. *******************************************************************************/ @ExtendWith(LocalstackDockerExtension.class) @LocalstackDockerProperties(services = { ServiceName.S3 }, portEdge = "2960", portElasticSearch = "2961") diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java new file mode 100644 index 00000000..e0cb0489 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java @@ -0,0 +1,151 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3; + + +import java.util.List; +import java.util.UUID; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for S3BackendModule + *******************************************************************************/ +public class S3BackendModuleTest extends BaseS3Test +{ + private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteFile() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // first list the files - then delete one, then re-list, and assert that we have one fewer // + ///////////////////////////////////////////////////////////////////////////////////////////// + List s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + + S3BackendModule s3BackendModule = new S3BackendModule(); + s3BackendModule.setS3Utils(getS3Utils()); + s3BackendModule.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey()); + + List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(), + "Should be one fewer file listed after deleting one."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteFileDoesNotExist() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count // + // note, we'd like to detect the non-delete, but there's no such info back from aws it appears? // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + + S3BackendModule s3BackendModule = new S3BackendModule(); + s3BackendModule.setS3Utils(getS3Utils()); + s3BackendModule.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + + List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size(), s3ObjectSummariesAfterDelete.size(), + "Should be same number of files after deleting bogus path"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMoveFile() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3); + String subPath = TEST_FOLDER + "/" + SUB_FOLDER; + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // first list the files (non-recursively) - then move one into a sub-folder, then re-list, and // + // assert that we have one fewer then list again including sub-folders, and see the changed count // + //////////////////////////////////////////////////////////////////////////////////////////////////// + List s3ObjectSummariesBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + List s3ObjectSummariesInSubFolderBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, subPath, false); + List s3ObjectSummariesRecursiveBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, true); + + S3BackendModule s3BackendModule = new S3BackendModule(); + s3BackendModule.setS3Utils(getS3Utils()); + String key = s3ObjectSummariesBeforeMove.get(0).getKey(); + s3BackendModule.moveFile(qInstance, table, key, key.replaceFirst(TEST_FOLDER, subPath)); + + List s3ObjectSummariesAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + List s3ObjectSummariesRecursiveAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, true); + List s3ObjectSummariesInSubFolderAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, subPath, false); + + Assertions.assertEquals(s3ObjectSummariesBeforeMove.size() - 1, s3ObjectSummariesAfterMove.size(), + "Should be one fewer file in the non-recursive listing after moving one."); + Assertions.assertEquals(s3ObjectSummariesRecursiveBeforeMove.size(), s3ObjectSummariesRecursiveAfterMove.size(), + "Should be same number of files in the recursive listing before and after the move"); + Assertions.assertEquals(s3ObjectSummariesInSubFolderBeforeMove.size() + 1, s3ObjectSummariesInSubFolderAfterMove.size(), + "Should be one move file in the sub-folder listing after moving one."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMoveFileDoesNotExit() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3); + String subPath = TEST_FOLDER + "/" + SUB_FOLDER; + + S3BackendModule s3BackendModule = new S3BackendModule(); + s3BackendModule.setS3Utils(getS3Utils()); + + Assertions.assertThrows(FilesystemException.class, () -> + s3BackendModule.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, subPath + "/" + UUID.randomUUID()) + ); + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java index 6fe9d2e0..ae490a44 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java @@ -27,6 +27,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -44,11 +45,14 @@ public class S3QueryActionTest extends BaseS3Test @Test public void testQuery1() throws QException { - QueryRequest queryRequest = initQueryRequest(); + QueryRequest queryRequest = initQueryRequest(); S3QueryAction s3QueryAction = new S3QueryAction(); s3QueryAction.setS3Utils(getS3Utils()); - QueryResult queryResult = s3QueryAction.execute(queryRequest); + QueryResult queryResult = s3QueryAction.execute(queryRequest); Assertions.assertEquals(5, queryResult.getRecords().size(), "Expected # of rows from unfiltered query"); + Assertions.assertTrue(queryResult.getRecords().stream() + .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseS3Test.TEST_FOLDER)), + "All records should have a full-path in their backend details, matching the test folder name"); }