diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ef02745..40a1a443 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,3 +1,6 @@ +## Deviations from qqq-java library standard circleci config: +## - To test AWS S3, uses localstsack executor, orb, and startup step +## - This docker image doesn't have java-17, so we install (and cache) jvm-17 version: 2.1 executors: @@ -8,8 +11,18 @@ executors: orbs: slack: circleci/slack@4.10.1 + localstack: localstack/platform@1.0 commands: + install_java17: + steps: + - run: + name: Install Java 17 + command: | + sudo add-apt-repository -y ppa:openjdk-r/ppa + sudo apt install -y openjdk-17-jdk + sudo rm /etc/alternatives/java + sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java run_maven: parameters: maven_subcommand: @@ -39,16 +52,20 @@ commands: jobs: mvn_test: - executor: java17 + executor: localstack/default steps: + - localstack/startup + - install_java17 - run_maven: maven_subcommand: test - slack/notify: event: fail mvn_deploy: - executor: java17 + executor: localstack/default steps: + - localstack/startup + - install_java17 - run_maven: maven_subcommand: deploy - slack/notify: diff --git a/pom.xml b/pom.xml index 51e09ceb..83edccf9 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ com.kingsrook.qqq qqq-backend-module-filesystem - 0.0.0-SNAPSHOT + 0.0.0 scm:git:git@github.com:Kingsrook/qqq-backend-module-rdbms.git @@ -51,7 +51,7 @@ com.kingsrook.qqq qqq-backend-core - 0.0.0-SNAPSHOT + 0.0.0 @@ -61,21 +61,9 @@ 1.12.243 - org.testcontainers - testcontainers - 1.17.2 - test - - - org.testcontainers - localstack - 1.17.2 - test - - - org.testcontainers - junit-jupiter - 1.17.2 + cloud.localstack + localstack-utils + 0.2.20 test diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java new file mode 100644 index 00000000..3051e18a --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java @@ -0,0 +1,39 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base; + + +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; + + +/******************************************************************************* + ** Interface to add additional functionality commonly among the various filesystem + ** module implementations. + *******************************************************************************/ +public interface FilesystemBackendModuleInterface +{ + /******************************************************************************* + ** For filesystem backends, get the module-specific action base-class, that helps + ** with functions like listing and deleting files. + *******************************************************************************/ + AbstractBaseFilesystemAction getActionBase(); +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemRecordBackendDetailFields.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemRecordBackendDetailFields.java new file mode 100644 index 00000000..415aa06f --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemRecordBackendDetailFields.java @@ -0,0 +1,32 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base; + + +/******************************************************************************* + ** Define the field names (keys) to be used in the backendDetails structure + ** under Records from this (these) modules. + *******************************************************************************/ +public interface FilesystemRecordBackendDetailFields +{ + String FULL_PATH = "fullPath"; +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java new file mode 100644 index 00000000..9a04f289 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -0,0 +1,294 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.actions; + + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; +import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.NotImplementedException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Base class for all Filesystem actions across all modules. + ** + ** @param FILE The class that represents a file in the sub-module. e.g., + * a java.io.File, or an S3Object. + *******************************************************************************/ +public abstract class AbstractBaseFilesystemAction +{ + private static final Logger LOG = LogManager.getLogger(AbstractBaseFilesystemAction.class); + + + + /******************************************************************************* + ** List the files for a table - to be implemented in module-specific subclasses. + *******************************************************************************/ + public abstract List listFiles(QTableMetaData table, QBackendMetaData backendBase); + + /******************************************************************************* + ** Read the contents of a file - to be implemented in module-specific subclasses. + *******************************************************************************/ + public abstract InputStream readFile(FILE file) throws IOException; + + /******************************************************************************* + ** Write a file - to be implemented in module-specific subclasses. + *******************************************************************************/ + public abstract void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException; + + /******************************************************************************* + ** Get a string that represents the full path to a file. + *******************************************************************************/ + public abstract String getFullPathForFile(FILE file); + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + *******************************************************************************/ + public abstract void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException; + + /******************************************************************************* + ** Move a file from a source path, to a destination path. + ** + ** @throws FilesystemException if the move is known to have failed + *******************************************************************************/ + public abstract void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException; + + /******************************************************************************* + ** e.g., with a base path of /foo/ + ** and a table path of /bar/ + ** and a file at /foo/bar/baz.txt + ** give us just the baz.txt part. + *******************************************************************************/ + public abstract String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData sourceBackend, QTableMetaData sourceTable); + + + + /******************************************************************************* + ** Append together the backend's base path (if present), with a table's base + ** path (again, if present). + *******************************************************************************/ + public String getFullBasePath(QTableMetaData table, QBackendMetaData backendBase) + { + AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase); + String fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : ""; + + AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); + if(StringUtils.hasContent(tableDetails.getBasePath())) + { + fullPath += File.separatorChar + tableDetails.getBasePath(); + } + + fullPath += File.separatorChar; + fullPath = stripDuplicatedSlashes(fullPath); + + return fullPath; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String stripDuplicatedSlashes(String path) + { + if(path == null) + { + return (null); + } + + return (path.replaceAll("//+", "/")); + } + + + + /******************************************************************************* + ** Get the backend metaData, type-checked as the requested type. + *******************************************************************************/ + protected T getBackendMetaData(Class outputClass, QBackendMetaData metaData) + { + if(!(outputClass.isInstance(metaData))) + { + throw new IllegalArgumentException("MetaData was not of expected type (was " + metaData.getClass().getSimpleName() + ")"); + } + return outputClass.cast(metaData); + } + + + + /******************************************************************************* + ** Get the backendDetails out of a table, type-checked as the requested type + *******************************************************************************/ + protected T getTableBackendDetails(Class outputClass, QTableMetaData tableMetaData) + { + QTableBackendDetails tableBackendDetails = tableMetaData.getBackendDetails(); + if(!(outputClass.isInstance(tableBackendDetails))) + { + throw new IllegalArgumentException("Table backend details was not of expected type (was " + tableBackendDetails.getClass().getSimpleName() + ")"); + } + return outputClass.cast(tableBackendDetails); + } + + + + /******************************************************************************* + ** Generic implementation of the execute method from the QueryInterface + *******************************************************************************/ + public QueryResult executeQuery(QueryRequest queryRequest) throws QException + { + preAction(queryRequest.getBackend()); + + try + { + QueryResult rs = new QueryResult(); + List records = new ArrayList<>(); + rs.setRecords(records); + + QTableMetaData table = queryRequest.getTable(); + AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); + List files = listFiles(table, queryRequest.getBackend()); + + for(FILE file : files) + { + LOG.info("Processing file: " + getFullPathForFile(file)); + switch(tableDetails.getRecordFormat()) + { + case CSV: + { + String fileContents = IOUtils.toString(readFile(file)); + fileContents = customizeFileContentsAfterReading(table, fileContents); + + List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + + records.addAll(recordsInFile); + break; + } + case JSON: + { + String fileContents = IOUtils.toString(readFile(file)); + fileContents = customizeFileContentsAfterReading(table, fileContents); + + List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + + records.addAll(recordsInFile); + break; + } + default: + { + throw new NotImplementedException("Filesystem record format " + tableDetails.getRecordFormat() + " is not yet implemented"); + } + } + } + + return rs; + } + catch(Exception e) + { + LOG.warn("Error executing query", e); + throw new QException("Error executing query", e); + } + } + + + + /******************************************************************************* + ** Add backend details to records about the file that they are in. + *******************************************************************************/ + protected void addBackendDetailsToRecords(List recordsInFile, FILE file) + { + recordsInFile.forEach(record -> + { + record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, getFullPathForFile(file)); + }); + } + + + + /******************************************************************************* + ** Method that subclasses can override to add pre-action things (e.g., setting up + ** s3 client). + *******************************************************************************/ + public void preAction(QBackendMetaData backendMetaData) + { + ///////////////////////////////////////////////////////////////////// + // noop in base class - subclasses can add functionality if needed // + ///////////////////////////////////////////////////////////////////// + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String customizeFileContentsAfterReading(QTableMetaData table, String fileContents) throws QException + { + Optional optionalCustomizer = table.getCustomizer("postFileRead"); + if(optionalCustomizer.isEmpty()) + { + return (fileContents); + } + QCodeReference customizer = optionalCustomizer.get(); + + try + { + Class customizerClass = Class.forName(customizer.getName()); + + @SuppressWarnings("unchecked") + Function function = (Function) customizerClass.getConstructor().newInstance(); + + return function.apply(fileContents); + } + catch(Exception e) + { + throw (new QException("Error customizing file contents", e)); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java new file mode 100644 index 00000000..9002bdb8 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java @@ -0,0 +1,80 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; + + +/******************************************************************************* + ** Base class for all BackendMetaData for all filesystem-style backend modules. + *******************************************************************************/ +public class AbstractFilesystemBackendMetaData extends QBackendMetaData +{ + private String basePath; + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public AbstractFilesystemBackendMetaData() + { + super(); + } + + + + /******************************************************************************* + ** Getter for basePath + ** + *******************************************************************************/ + public String getBasePath() + { + return (basePath); + } + + + + /******************************************************************************* + ** Setter for basePath + ** + *******************************************************************************/ + public void setBasePath(String basePath) + { + this.basePath = basePath; + } + + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withBasePath(String basePath) + { + this.basePath = basePath; + return (T) this; + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java new file mode 100644 index 00000000..e98e4fad --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java @@ -0,0 +1,178 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; + + +/******************************************************************************* + ** Extension of QTableBackendDetails, with details specific to a Filesystem table. + *******************************************************************************/ +public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails +{ + private String basePath; + private String glob; + private RecordFormat recordFormat; + private Cardinality cardinality; + + + + /******************************************************************************* + ** Getter for basePath + ** + *******************************************************************************/ + public String getBasePath() + { + return basePath; + } + + + + /******************************************************************************* + ** Setter for basePath + ** + *******************************************************************************/ + public void setBasePath(String basePath) + { + this.basePath = basePath; + } + + + + /******************************************************************************* + ** Fluent Setter for basePath + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withBasePath(String basePath) + { + this.basePath = basePath; + return (T) this; + } + + + + /******************************************************************************* + ** Getter for glob + ** + *******************************************************************************/ + public String getGlob() + { + return glob; + } + + + + /******************************************************************************* + ** Setter for glob + ** + *******************************************************************************/ + public void setGlob(String glob) + { + this.glob = glob; + } + + + + /******************************************************************************* + ** Fluent Setter for glob + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withGlob(String glob) + { + this.glob = glob; + return (T) this; + } + + + + /******************************************************************************* + ** Getter for recordFormat + ** + *******************************************************************************/ + public RecordFormat getRecordFormat() + { + return recordFormat; + } + + + + /******************************************************************************* + ** Setter for recordFormat + ** + *******************************************************************************/ + public void setRecordFormat(RecordFormat recordFormat) + { + this.recordFormat = recordFormat; + } + + + + /******************************************************************************* + ** Fluent Setter for recordFormat + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withRecordFormat(RecordFormat recordFormat) + { + this.recordFormat = recordFormat; + return ((T) this); + } + + + + /******************************************************************************* + ** Getter for cardinality + ** + *******************************************************************************/ + public Cardinality getCardinality() + { + return cardinality; + } + + + + /******************************************************************************* + ** Setter for cardinality + ** + *******************************************************************************/ + public void setCardinality(Cardinality cardinality) + { + this.cardinality = cardinality; + } + + + + /******************************************************************************* + ** Fluent Setter for cardinality + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withCardinality(Cardinality cardinality) + { + this.cardinality = cardinality; + return ((T) this); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/Cardinality.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/Cardinality.java new file mode 100644 index 00000000..0fae2ae6 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/Cardinality.java @@ -0,0 +1,33 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; + + +/******************************************************************************* + ** Mode for filesystem backends: are all records in a single file, or are there + ** many files? + *******************************************************************************/ +public enum Cardinality +{ + ONE, + MANY +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/RecordFormat.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/RecordFormat.java new file mode 100644 index 00000000..572876de --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/RecordFormat.java @@ -0,0 +1,34 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; + + +/******************************************************************************* + ** How are records stored in the files in a filesystem backend? CSV, JSON, + ** future types may XML, or more exotic ones, like "BINARY" or "TEXT" (e.g., 1 + ** record and 1 field per-file) + *******************************************************************************/ +public enum RecordFormat +{ + CSV, + JSON +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/exceptions/FilesystemException.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/exceptions/FilesystemException.java new file mode 100644 index 00000000..cbda757b --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/exceptions/FilesystemException.java @@ -0,0 +1,52 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.exceptions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; + + +/******************************************************************************* + ** Checked exception to be thrown from actions within this module. + *******************************************************************************/ +public class FilesystemException extends QException +{ + /******************************************************************************* + ** Constructor of message + ** + *******************************************************************************/ + public FilesystemException(String message) + { + super(message); + } + + + + /******************************************************************************* + ** Constructor of message & cause + ** + *******************************************************************************/ + public FilesystemException(String message, Throwable cause) + { + super(message, cause); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java new file mode 100644 index 00000000..f432ec7b --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java @@ -0,0 +1,142 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local; + + +import java.io.File; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemUpdateAction; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** QQQ Backend module for working with (local) Filesystems. + *******************************************************************************/ +public class FilesystemBackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface +{ + private static final Logger LOG = LogManager.getLogger(FilesystemBackendModule.class); + + + + /******************************************************************************* + ** For filesystem backends, get the module-specific action base-class, that helps + ** with functions like listing and deleting files. + *******************************************************************************/ + @Override + public AbstractBaseFilesystemAction getActionBase() + { + return (new AbstractFilesystemAction()); + } + + + + /******************************************************************************* + ** Method where a backend module must be able to provide its type (name). + *******************************************************************************/ + @Override + public String getBackendType() + { + return ("filesystem"); + } + + + + /******************************************************************************* + ** Method to identify the class used for backend meta data for this module. + *******************************************************************************/ + @Override + public Class getBackendMetaDataClass() + { + return (FilesystemBackendMetaData.class); + } + + + + /******************************************************************************* + ** Method to identify the class used for table-backend details for this module. + *******************************************************************************/ + @Override + public Class getTableBackendDetailsClass() + { + return FilesystemTableBackendDetails.class; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryInterface getQueryInterface() + { + return new FilesystemQueryAction(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InsertInterface getInsertInterface() + { + return (new FilesystemInsertAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public UpdateInterface getUpdateInterface() + { + return (new FilesystemUpdateAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public DeleteInterface getDeleteInterface() + { + return (new FilesystemDeleteAction()); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java new file mode 100644 index 00000000..be5dd44b --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -0,0 +1,186 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Base class for all (local) Filesystem actions + *******************************************************************************/ +public class AbstractFilesystemAction extends AbstractBaseFilesystemAction +{ + private static final Logger LOG = LogManager.getLogger(AbstractFilesystemAction.class); + + + + /******************************************************************************* + ** List the files for this table. + *******************************************************************************/ + @Override + public List listFiles(QTableMetaData table, QBackendMetaData backendBase) + { + // todo - needs rewritten to do globbing... + String fullPath = getFullBasePath(table, backendBase); + File directory = new File(fullPath); + File[] files = directory.listFiles(); + + if(files == null) + { + return Collections.emptyList(); + } + + return (Arrays.stream(files).filter(File::isFile).toList()); + } + + + + /******************************************************************************* + ** Read the contents of a file. + *******************************************************************************/ + @Override + public InputStream readFile(File file) throws IOException + { + return (new FileInputStream(file)); + } + + + + /******************************************************************************* + ** Write a file - to be implemented in module-specific subclasses. + *******************************************************************************/ + @Override + public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException + { + FileUtils.writeByteArrayToFile(new File(path), contents); + } + + + + /******************************************************************************* + ** Get a string that represents the full path to a file. + *******************************************************************************/ + @Override + public String getFullPathForFile(File file) + { + return (file.getAbsolutePath()); + } + + + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + *******************************************************************************/ + @Override + public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + { + File file = new File(fileReference); + if(!file.exists()) + { + ////////////////////////////////////////////////////////////////////////////////////////////// + // if the file doesn't exist, just exit with noop. don't throw an error - that should only // + // happen if the "contract" of the method is broken, and the file still exists // + ////////////////////////////////////////////////////////////////////////////////////////////// + LOG.debug("Not deleting file [{}], because it does not exist.", file); + return; + } + + if(!file.delete()) + { + throw (new FilesystemException("Failed to delete file: " + fileReference)); + } + } + + + + /******************************************************************************* + ** Move a file from a source path, to a destination path. + ** + ** @param destination assumed to be a file path - not a directory + ** @throws FilesystemException if the delete is known to have failed + *******************************************************************************/ + @Override + public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException + { + File sourceFile = new File(source); + File destinationFile = new File(destination); + File destinationParent = destinationFile.getParentFile(); + + if(!sourceFile.exists()) + { + throw (new FilesystemException("Cannot move file " + source + ", as it does not exist.")); + } + + ////////////////////////////////////////////////////////////////////////////////////// + // if the destination folder doesn't exist, try to make it - and fail if that fails // + ////////////////////////////////////////////////////////////////////////////////////// + if(!destinationParent.exists()) + { + LOG.debug("Making destination directory {} for move", destinationParent.getAbsolutePath()); + if(!destinationParent.mkdirs()) + { + throw (new FilesystemException("Failed to make destination directory " + destinationParent.getAbsolutePath() + " to move " + source + " into.")); + } + } + + if(!sourceFile.renameTo(destinationFile)) + { + throw (new FilesystemException("Failed to move (rename) file " + source + " to " + destination)); + } + } + + + + /******************************************************************************* + ** e.g., with a base path of /foo/ + ** and a table path of /bar/ + ** and a file at /foo/bar/baz.txt + ** give us just the baz.txt part. + *******************************************************************************/ + @Override + public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table) + { + String tablePath = getFullBasePath(table, backend); + String strippedPath = filePath.replaceFirst(".*" + tablePath, ""); + return (strippedPath); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java new file mode 100644 index 00000000..9843bb4e --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java @@ -0,0 +1,60 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest; +import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult; +import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilesystemDeleteAction implements DeleteInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public DeleteResult execute(DeleteRequest deleteRequest) throws QException + { + throw new NotImplementedException("Filesystem delete not implemented"); + /* + try + { + DeleteResult rs = new DeleteResult(); + QTableMetaData table = deleteRequest.getTable(); + + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing delete: " + e.getMessage(), e); + } + */ + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java new file mode 100644 index 00000000..9a8ca832 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest; +import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult; +import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilesystemInsertAction implements InsertInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public InsertResult execute(InsertRequest insertRequest) throws QException + { + throw new NotImplementedException("Filesystem insert not implemented"); + /* + try + { + InsertResult rs = new InsertResult(); + QTableMetaData table = insertRequest.getTable(); + + List recordsWithStatus = new ArrayList<>(); + rs.setRecords(recordsWithStatus); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing insert: " + e.getMessage(), e); + } + */ + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryAction.java new file mode 100644 index 00000000..56dc6d5d --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; +import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilesystemQueryAction extends AbstractFilesystemAction implements QueryInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public QueryResult execute(QueryRequest queryRequest) throws QException + { + return executeQuery(queryRequest); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java new file mode 100644 index 00000000..1f762164 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java @@ -0,0 +1,63 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest; +import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult; +import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilesystemUpdateAction implements UpdateInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public UpdateResult execute(UpdateRequest updateRequest) throws QException + { + throw new NotImplementedException("Filesystem update not implemented"); + /* + try + { + UpdateResult rs = new UpdateResult(); + QTableMetaData table = updateRequest.getTable(); + + List records = new ArrayList<>(); + rs.setRecords(records); + + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing update: " + e.getMessage(), e); + } + */ + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java new file mode 100644 index 00000000..02b3950e --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule; + + +/******************************************************************************* + ** (local) Filesystem backend meta data. + *******************************************************************************/ +public class FilesystemBackendMetaData extends AbstractFilesystemBackendMetaData +{ + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public FilesystemBackendMetaData() + { + super(); + setBackendType(FilesystemBackendModule.class); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemTableBackendDetails.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemTableBackendDetails.java new file mode 100644 index 00000000..b2f8cee9 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemTableBackendDetails.java @@ -0,0 +1,44 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule; + + +/******************************************************************************* + ** (local) Filesystem specific Extension of QTableBackendDetails + *******************************************************************************/ +public class FilesystemTableBackendDetails extends AbstractFilesystemTableBackendDetails +{ + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public FilesystemTableBackendDetails() + { + super(); + setBackendType(FilesystemBackendModule.class); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java new file mode 100644 index 00000000..f91a5b10 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java @@ -0,0 +1,135 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic; + + +import java.io.File; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Function body for performing the Cleanup step of a basic ETL process - e.g., + ** after the loading, delete or move the processed file(s). + *******************************************************************************/ +public class BasicETLCleanupSourceFilesFunction implements FunctionBody +{ + private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesFunction.class); + + public static final String FIELD_MOVE_OR_DELETE = "moveOrDelete"; + public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves"; + + public static final String VALUE_MOVE = "move"; + public static final String VALUE_DELETE = "delete"; + public static final String FUNCTION_NAME = "cleanupSourceFiles"; + + + + /******************************************************************************* + ** Execute the function - using the request as input, and the result as output. + *******************************************************************************/ + @Override + public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + { + String sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE); + QTableMetaData table = runFunctionRequest.getInstance().getTable(sourceTableName); + QBackendMetaData backend = runFunctionRequest.getInstance().getBackendForTable(sourceTableName); + QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend); + + if(!(module instanceof FilesystemBackendModuleInterface filesystemModule)) + { + throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function.")); + } + AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase(); + actionBase.preAction(backend); + + String sourceFilePaths = runFunctionRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS); + if(!StringUtils.hasContent(sourceFilePaths)) + { + LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS + "]"); + return; + } + + String[] sourceFiles = sourceFilePaths.split(","); + for(String sourceFile : sourceFiles) + { + String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE); + if(VALUE_DELETE.equals(moveOrDelete)) + { + actionBase.deleteFile(runFunctionRequest.getInstance(), table, sourceFile); + } + else if(VALUE_MOVE.equals(moveOrDelete)) + { + String destinationForMoves = runFunctionRequest.getValueString(FIELD_DESTINATION_FOR_MOVES); + if(!StringUtils.hasContent(destinationForMoves)) + { + throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value.")); + } + String filePathWithoutBase = actionBase.stripBackendAndTableBasePathsFromFileName(sourceFile, backend, table); + String destinationPath = destinationForMoves + File.separator + filePathWithoutBase; + actionBase.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath); + } + else + { + throw (new QException("Unexpected value [" + moveOrDelete + "] for field [" + FIELD_MOVE_OR_DELETE + "]. " + + "Must be either [" + VALUE_MOVE + "] or [" + VALUE_DELETE + "].")); + } + } + } + + + + /******************************************************************************* + ** define the metaData that describes this function + *******************************************************************************/ + public QFunctionMetaData defineFunctionMetaData() + { + return (new QFunctionMetaData() + .withName(FUNCTION_NAME) + .withCode(new QCodeReference() + .withName(this.getClass().getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.FUNCTION)) + .withInputData(new QFunctionInputMetaData() + .addField(new QFieldMetaData("moveOrDelete", QFieldType.STRING)) + .addField(new QFieldMetaData("destinationForMoves", QFieldType.STRING)))); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java new file mode 100644 index 00000000..20de530f --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java @@ -0,0 +1,84 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic; + + +import java.util.Set; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; + + +/******************************************************************************* + ** Function body for collecting the file names that were discovered in the + ** Extract step. These will be lost during the transform, so we capture them here, + ** so that our Clean function can move or delete them. + ** + ** TODO - need unit test!! + *******************************************************************************/ +public class BasicETLCollectSourceFileNamesFunction implements FunctionBody +{ + public static final String FUNCTION_NAME = "collectSourceFileNames"; + public static final String FIELD_SOURCE_FILE_PATHS = "sourceFilePaths"; + + + + /******************************************************************************* + ** Execute the function - using the request as input, and the result as output. + *******************************************************************************/ + @Override + public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + { + Set sourceFiles = runFunctionRequest.getRecords().stream() + .map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH)) + .collect(Collectors.toSet()); + runFunctionResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles)); + } + + + + /******************************************************************************* + ** define the metaData that describes this function + *******************************************************************************/ + public QFunctionMetaData defineFunctionMetaData() + { + return (new QFunctionMetaData() + .withName(FUNCTION_NAME) + .withCode(new QCodeReference() + .withName(this.getClass().getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.FUNCTION)) + .withOutputMetaData(new QFunctionOutputMetaData() + .addField(new QFieldMetaData(FIELD_SOURCE_FILE_PATHS, QFieldType.STRING)))); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java new file mode 100644 index 00000000..45264376 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java @@ -0,0 +1,138 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync; + + +import java.io.File; +import java.io.InputStream; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Function body for collecting the file names that were discovered in the + ** Extract step. These will be lost during the transform, so we capture them here, + ** so that our Clean function can move or delete them. + ** + *******************************************************************************/ +public class FilesystemSyncFunction implements FunctionBody +{ + private static final Logger LOG = LogManager.getLogger(FilesystemSyncFunction.class); + + public static final String FUNCTION_NAME = "sync"; + + + + /******************************************************************************* + ** Execute the function - using the request as input, and the result as output. + *******************************************************************************/ + @Override + public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + { + QTableMetaData sourceTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE)); + QTableMetaData archiveTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE)); + QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE)); + + QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName()); + FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend); + AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase(); + sourceActionBase.preAction(sourceBackend); + Map sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend); + + QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName()); + FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend); + AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase(); + archiveActionBase.preAction(archiveBackend); + Set archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet(); + + QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName()); + FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend); + AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase(); + processingActionBase.preAction(processingBackend); + + Integer maxFilesToSync = runFunctionRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE); + int syncedFileCount = 0; + for(Map.Entry sourceEntry : sourceFiles.entrySet()) + { + try + { + String sourceFileName = sourceEntry.getKey(); + if(!archiveFiles.contains(sourceFileName)) + { + LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]"); + InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue()); + byte[] bytes = inputStream.readAllBytes(); + + String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend); + archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes); + + String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend); + processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes); + syncedFileCount++; + + if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync) + { + LOG.info("Breaking after syncing " + syncedFileCount + " files"); + break; + } + } + } + catch(Exception e) + { + LOG.error("Error processing file: " + sourceEntry, e); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private Map getFileNames(AbstractBaseFilesystemAction actionBase, QTableMetaData table, QBackendMetaData backend) + { + List files = actionBase.listFiles(table, backend); + Map rs = new LinkedHashMap<>(); + + for(Object file : files) + { + String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table); + rs.put(fileName, file); + } + + return (rs); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java new file mode 100644 index 00000000..5773630a --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java @@ -0,0 +1,83 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync; + + +import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; + + +/******************************************************************************* + ** Definition for Filesystem sync process. + ** + ** Job is to: + ** - list all files in the source table. + ** - list all files in the archive table. + ** - if any files exist in the source, but not in the archive, then: + ** - copy the file to both the archive and the processing table. + ** + ** The maxFilesToArchive field can be used to only sync up to that many files + ** (an help an initial sync, if you want to do it in smaller batches) + ** + ** The idea being, that the source is read-only, and we want to move files out of + ** processing after they've been processed - and the archive is what we can have + ** in-between the two. + *******************************************************************************/ +public class FilesystemSyncProcess +{ + public static final String PROCESS_NAME = "filesystem.sync"; + + public static final String FIELD_SOURCE_TABLE = "sourceTable"; + public static final String FIELD_ARCHIVE_TABLE = "archiveTable"; + public static final String FIELD_PROCESSING_TABLE = "processingTable"; + public static final String FIELD_MAX_FILES_TO_ARCHIVE = "maxFilesToArchive"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QProcessMetaData defineProcessMetaData() + { + QFunctionMetaData syncFunction = new QFunctionMetaData() + .withName(FilesystemSyncFunction.FUNCTION_NAME) + .withCode(new QCodeReference() + .withName(FilesystemSyncFunction.class.getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.FUNCTION)) + .withInputData(new QFunctionInputMetaData() + .addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)) + .addField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING)) + .addField(new QFieldMetaData(FIELD_MAX_FILES_TO_ARCHIVE, QFieldType.INTEGER).withDefaultValue(Integer.MAX_VALUE)) + .addField(new QFieldMetaData(FIELD_PROCESSING_TABLE, QFieldType.STRING))); + + return new QProcessMetaData() + .withName(PROCESS_NAME) + .addFunction(syncFunction); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java new file mode 100644 index 00000000..38a54535 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java @@ -0,0 +1,138 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3; + + +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; + + +/******************************************************************************* + ** QQQ Backend module for working with AWS S3 filesystems + *******************************************************************************/ +public class S3BackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface +{ + + + /******************************************************************************* + ** For filesystem backends, get the module-specific action base-class, that helps + ** with functions like listing and deleting files. + *******************************************************************************/ + @Override + public AbstractBaseFilesystemAction getActionBase() + { + return (new AbstractS3Action()); + } + + + + /******************************************************************************* + ** Method where a backend module must be able to provide its type (name). + *******************************************************************************/ + @Override + public String getBackendType() + { + return ("s3"); + } + + + + /******************************************************************************* + ** Method to identify the class used for backend meta data for this module. + *******************************************************************************/ + @Override + public Class getBackendMetaDataClass() + { + return (S3BackendMetaData.class); + } + + + + /******************************************************************************* + ** Method to identify the class used for table-backend details for this module. + *******************************************************************************/ + @Override + public Class getTableBackendDetailsClass() + { + return (S3TableBackendDetails.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryInterface getQueryInterface() + { + return new S3QueryAction(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InsertInterface getInsertInterface() + { + return (new S3InsertAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public UpdateInterface getUpdateInterface() + { + return (new S3UpdateAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public DeleteInterface getDeleteInterface() + { + return (new S3DeleteAction()); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java new file mode 100644 index 00000000..21958ab7 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -0,0 +1,237 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Base class for all S3 filesystem actions + *******************************************************************************/ +public class AbstractS3Action extends AbstractBaseFilesystemAction +{ + private static final Logger LOG = LogManager.getLogger(AbstractS3Action.class); + + private S3Utils s3Utils; + + + + /******************************************************************************* + ** Setup the s3 utils object to be used for this action. + *******************************************************************************/ + @Override + public void preAction(QBackendMetaData backendMetaData) + { + super.preAction(backendMetaData); + + if(s3Utils != null) + { + LOG.debug("s3Utils object is already set - not re-setting it."); + return; + } + + S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendMetaData); + AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard(); + + if(StringUtils.hasContent(s3BackendMetaData.getAccessKey()) && StringUtils.hasContent(s3BackendMetaData.getSecretKey())) + { + BasicAWSCredentials credentials = new BasicAWSCredentials(s3BackendMetaData.getAccessKey(), s3BackendMetaData.getSecretKey()); + clientBuilder.setCredentials(new AWSStaticCredentialsProvider(credentials)); + } + + if(StringUtils.hasContent(s3BackendMetaData.getRegion())) + { + clientBuilder.setRegion(s3BackendMetaData.getRegion()); + } + + s3Utils = new S3Utils(); + s3Utils.setAmazonS3(clientBuilder.build()); + } + + + + /******************************************************************************* + ** Set the S3Utils object. + *******************************************************************************/ + public void setS3Utils(S3Utils s3Utils) + { + this.s3Utils = s3Utils; + } + + + + /******************************************************************************* + ** Internal accessor for the s3Utils object - should always use this, not the field. + *******************************************************************************/ + private S3Utils getS3Utils() + { + if(s3Utils == null) + { + s3Utils = new S3Utils(); + } + + return s3Utils; + } + + + + /******************************************************************************* + ** List the files for a table. + *******************************************************************************/ + @Override + public List listFiles(QTableMetaData table, QBackendMetaData backendBase) + { + S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase); + AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); + + String fullPath = getFullBasePath(table, backendBase); + String bucketName = s3BackendMetaData.getBucketName(); + String glob = tableDetails.getGlob(); + + //////////////////////////////////////////////////////////////////// + // todo - look at metadata to configure the s3 client here? // + //////////////////////////////////////////////////////////////////// + return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob); + } + + + + /******************************************************************************* + ** Read the contents of a file. + *******************************************************************************/ + @Override + public InputStream readFile(S3ObjectSummary s3ObjectSummary) throws IOException + { + return (getS3Utils().getObjectAsInputStream(s3ObjectSummary)); + } + + + + /******************************************************************************* + ** Write a file - to be implemented in module-specific subclasses. + *******************************************************************************/ + @Override + public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException + { + path = stripLeadingSlash(stripDuplicatedSlashes(path)); + String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName(); + getS3Utils().writeFile(bucketName, path, contents); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String stripLeadingSlash(String path) + { + if(path == null) + { + return (null); + } + return (path.replaceFirst("^/+", "")); + } + + + + /******************************************************************************* + ** Get a string that represents the full path to a file. + *******************************************************************************/ + @Override + public String getFullPathForFile(S3ObjectSummary s3ObjectSummary) + { + return (s3ObjectSummary.getKey()); + } + + + + /******************************************************************************* + ** e.g., with a base path of /foo/ + ** and a table path of /bar/ + ** and a file at /foo/bar/baz.txt + ** give us just the baz.txt part. + *******************************************************************************/ + @Override + public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table) + { + String tablePath = getFullBasePath(table, backend); + String strippedPath = filePath.replaceFirst("^/*" + stripLeadingSlash(tablePath), ""); + return (strippedPath); + } + + + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + *******************************************************************************/ + @Override + public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + { + QBackendMetaData backend = instance.getBackend(table.getBackendName()); + String bucketName = ((S3BackendMetaData) backend).getBucketName(); + + getS3Utils().deleteObject(bucketName, fileReference); + } + + + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @param destination assumed to be a file path - not a directory + ** @throws FilesystemException if the move is known to have failed + *******************************************************************************/ + @Override + public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException + { + QBackendMetaData backend = instance.getBackend(table.getBackendName()); + String bucketName = ((S3BackendMetaData) backend).getBucketName(); + destination = stripLeadingSlash(stripDuplicatedSlashes(destination)); + + getS3Utils().moveObject(bucketName, source, destination); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java new file mode 100644 index 00000000..2a4a8448 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java @@ -0,0 +1,60 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest; +import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult; +import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3DeleteAction implements DeleteInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public DeleteResult execute(DeleteRequest deleteRequest) throws QException + { + throw new NotImplementedException("S3 delete not implemented"); + /* + try + { + DeleteResult rs = new DeleteResult(); + QTableMetaData table = deleteRequest.getTable(); + + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing delete: " + e.getMessage(), e); + } + */ + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java new file mode 100644 index 00000000..caeed82b --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java @@ -0,0 +1,63 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest; +import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult; +import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3InsertAction implements InsertInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public InsertResult execute(InsertRequest insertRequest) throws QException + { + throw new NotImplementedException("S3 insert not implemented"); + /* + try + { + InsertResult rs = new InsertResult(); + QTableMetaData table = insertRequest.getTable(); + + List recordsWithStatus = new ArrayList<>(); + rs.setRecords(recordsWithStatus); + + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing insert: " + e.getMessage(), e); + } + */ + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryAction.java new file mode 100644 index 00000000..46e71e25 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; +import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3QueryAction extends AbstractS3Action implements QueryInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public QueryResult execute(QueryRequest queryRequest) throws QException + { + return (super.executeQuery(queryRequest)); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java new file mode 100644 index 00000000..4955ffb2 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest; +import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult; +import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3UpdateAction implements UpdateInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public UpdateResult execute(UpdateRequest updateRequest) throws QException + { + throw new NotImplementedException("S3 update not implemented"); + /* + try + { + UpdateResult rs = new UpdateResult(); + QTableMetaData table = updateRequest.getTable(); + + List records = new ArrayList<>(); + rs.setRecords(records); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing update: " + e.getMessage(), e); + } + */ + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java new file mode 100644 index 00000000..92dfccc1 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java @@ -0,0 +1,207 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata; + + +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; + + +/******************************************************************************* + ** S3 backend meta data. + *******************************************************************************/ +public class S3BackendMetaData extends AbstractFilesystemBackendMetaData +{ + private String bucketName; + private String accessKey; + private String secretKey; + private String region; + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public S3BackendMetaData() + { + super(); + setBackendType(S3BackendModule.class); + } + + + + /******************************************************************************* + ** Getter for bucketName + ** + *******************************************************************************/ + public String getBucketName() + { + return bucketName; + } + + + + /******************************************************************************* + ** Setter for bucketName + ** + *******************************************************************************/ + public void setBucketName(String bucketName) + { + this.bucketName = bucketName; + } + + + + /******************************************************************************* + ** Fluent setter for bucketName + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withBucketName(String bucketName) + { + this.bucketName = bucketName; + return (T) this; + } + + + + /******************************************************************************* + ** Getter for accessKey + ** + *******************************************************************************/ + public String getAccessKey() + { + return accessKey; + } + + + + /******************************************************************************* + ** Setter for accessKey + ** + *******************************************************************************/ + public void setAccessKey(String accessKey) + { + this.accessKey = accessKey; + } + + + + /******************************************************************************* + ** Fluent setter for accessKey + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withAccessKey(String accessKey) + { + this.accessKey = accessKey; + return (T) this; + } + + + + /******************************************************************************* + ** Getter for secretKey + ** + *******************************************************************************/ + public String getSecretKey() + { + return secretKey; + } + + + + /******************************************************************************* + ** Setter for secretKey + ** + *******************************************************************************/ + public void setSecretKey(String secretKey) + { + this.secretKey = secretKey; + } + + + + /******************************************************************************* + ** Fluent setter for secretKey + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withSecretKey(String secretKey) + { + this.secretKey = secretKey; + return (T) this; + } + + + + /******************************************************************************* + ** Getter for region + ** + *******************************************************************************/ + public String getRegion() + { + return region; + } + + + + /******************************************************************************* + ** Setter for region + ** + *******************************************************************************/ + public void setRegion(String region) + { + this.region = region; + } + + + + /******************************************************************************* + ** Fluent setter for region + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withRegion(String region) + { + this.region = region; + return (T) this; + } + + + + /******************************************************************************* + ** Called by the QInstanceEnricher - to do backend-type-specific enrichments. + ** Original use case is: reading secrets into fields (e.g., passwords). + ** TODO - migrate to use @InterpretableFields (and complete that impl on core side) + *******************************************************************************/ + @Override + public void enrich() + { + super.enrich(); + QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter(); + accessKey = interpreter.interpret(accessKey); + secretKey = interpreter.interpret(secretKey); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3TableBackendDetails.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3TableBackendDetails.java new file mode 100644 index 00000000..ba372cc6 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3TableBackendDetails.java @@ -0,0 +1,46 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; + + +/******************************************************************************* + ** S3 specific Extension of QTableBackendDetails + *******************************************************************************/ +public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails +{ + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public S3TableBackendDetails() + { + super(); + setBackendType(S3BackendModule.class); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java new file mode 100644 index 00000000..b791d355 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java @@ -0,0 +1,249 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.utils; + + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.util.ArrayList; +import java.util.List; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Utility methods for working with AWS S3. + ** + ** Note: May need a constructor (or similar) in the future that takes the + ** S3BackendMetaData - e.g., if we need some metaData to construct the AmazonS3 + ** (api client) object, such as region, or authentication. + *******************************************************************************/ +public class S3Utils +{ + private static final Logger LOG = LogManager.getLogger(S3Utils.class); + + private AmazonS3 amazonS3; + + + + /******************************************************************************* + ** List the objects in an S3 bucket matching a glob, per: + ** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String) + *******************************************************************************/ + public List listObjectsInBucketMatchingGlob(String bucketName, String path, String glob) + { + ////////////////////////////////////////////////////////////////////////////////////////////////// + // s3 list requests find nothing if the path starts with a /, so strip away any leading slashes // + // also strip away trailing /'s, for consistent known paths. // + // also normalize any duplicated /'s to a single /. // + ////////////////////////////////////////////////////////////////////////////////////////////////// + path = path.replaceFirst("^/+", "").replaceFirst("/+$", "").replaceAll("//+", "/"); + String prefix = path; + + // todo - maybe this is some error - that the user put a * in the path instead of the glob? + if(prefix.indexOf('*') > -1) + { + prefix = prefix.substring(0, prefix.indexOf('*')); + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob. // + // That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. // + // Anyway, add a trailing /* to globs with no wildcards (or just a '*' if it's a request for the root ("")) // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(glob == null) + { + glob = ""; + } + if(!glob.contains("*")) + { + if(glob.equals("")) + { + glob += "*"; + } + else + { + glob += "/*"; + } + } + + String pathMatcherArg = AbstractFilesystemAction.stripDuplicatedSlashes("glob:/" + path + "/" + glob); + PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher(pathMatcherArg); + + ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request() + .withBucketName(bucketName) + .withPrefix(prefix); + + ListObjectsV2Result listObjectsV2Result = null; + List rs = new ArrayList<>(); + + do + { + if(listObjectsV2Result != null) + { + listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken()); + } + LOG.info("Listing bucket=" + bucketName + ", path=" + path); + listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request); + + ////////////////////////////////// + // put files in the result list // + ////////////////////////////////// + for(S3ObjectSummary objectSummary : listObjectsV2Result.getObjectSummaries()) + { + String key = objectSummary.getKey(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // it looks like keys in s3 can have duplicated /'s - so normalize those, to create a "sane" result // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + key = key.replaceAll("//+", "/"); + + //////////////////////////////////////////////////////////////////////////////// + // always skip folders // + // this seemed to fire when it was first written, but not in our unit tests - // + // is this a difference with real s3 vs. localstack possibly? // + //////////////////////////////////////////////////////////////////////////////// + if(key.endsWith("/")) + { + LOG.debug("Skipping file [{}] because it is a folder", key); + continue; + } + + /////////////////////////////////////////// + // skip files that do not match the glob // + /////////////////////////////////////////// + if(!pathMatcher.matches(Path.of(URI.create("file:///" + key)))) + { + LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob); + continue; + } + + rs.add(objectSummary); + } + } + while(listObjectsV2Result.isTruncated()); + + return rs; + } + + + + /******************************************************************************* + ** Get the contents (as an InputStream) for an object in s3 + *******************************************************************************/ + public InputStream getObjectAsInputStream(S3ObjectSummary s3ObjectSummary) + { + return getAmazonS3().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()).getObjectContent(); + } + + + + /******************************************************************************* + ** Write a file + *******************************************************************************/ + public void writeFile(String bucket, String key, byte[] contents) + { + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentLength(contents.length); + getAmazonS3().putObject(bucket, key, new ByteArrayInputStream(contents), objectMetadata); + } + + + + /******************************************************************************* + ** Delete an object (file) from a bucket + *******************************************************************************/ + public void deleteObject(String bucketName, String key) throws FilesystemException + { + ////////////////////////////////////////////////////////////////////////////////////////////// + // note, aws s3 api does not appear to have any way to check the success or failure here... // + ////////////////////////////////////////////////////////////////////////////////////////////// + try + { + getAmazonS3().deleteObject(bucketName, key); + } + catch(Exception e) + { + throw (new FilesystemException("Error deleting s3 object " + key + " in bucket " + bucketName, e)); + } + } + + + + /******************************************************************************* + ** Move an object (file) within a bucket + *******************************************************************************/ + public void moveObject(String bucketName, String source, String destination) throws FilesystemException + { + ////////////////////////////////////////////////////////////////////////////////////////////// + // note, aws s3 api does not appear to have any way to check the success or failure here... // + ////////////////////////////////////////////////////////////////////////////////////////////// + try + { + getAmazonS3().copyObject(bucketName, source, bucketName, destination); + getAmazonS3().deleteObject(bucketName, source); + } + catch(Exception e) + { + throw (new FilesystemException("Error moving s3 object " + source + " to " + destination + " in bucket " + bucketName, e)); + } + } + + + + /******************************************************************************* + ** Setter for AmazonS3 client object. + *******************************************************************************/ + public void setAmazonS3(AmazonS3 amazonS3) + { + this.amazonS3 = amazonS3; + } + + + + /******************************************************************************* + ** Getter for AmazonS3 client object. + *******************************************************************************/ + public AmazonS3 getAmazonS3() + { + if(amazonS3 == null) + { + // TODO - get this (and other props?) from backend meta data + amazonS3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build(); + } + + return amazonS3; + } +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java new file mode 100644 index 00000000..676e3f96 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -0,0 +1,227 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem; + + +import java.io.File; +import java.io.IOException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.mock.MockAuthenticationModule; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import org.apache.commons.io.FileUtils; + + +/******************************************************************************* + ** Utility methods for Filesystem module tests + *******************************************************************************/ +public class TestUtils +{ + public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; + public static final String BACKEND_NAME_S3 = "s3"; + public static final String TABLE_NAME_PERSON_LOCAL_FS = "person"; + public static final String TABLE_NAME_PERSON_S3 = "person-s3"; + + /////////////////////////////////////////////////////////////////// + // shouldn't be accessed directly, as we append a counter to it. // + /////////////////////////////////////////////////////////////////// + public static final String BASE_PATH = "/tmp/filesystem-tests"; + + ////////////////////////////////////////////////////////////////////////////// + // Used to make each test method have a unique folder path, more or less... // + // See methods that work with it. // + ////////////////////////////////////////////////////////////////////////////// + private static int testInstanceCounter = 0; + + + + /******************************************************************************* + ** Meant to be called in a @BeforeEach - increment an internal counter that will + ** give us a unique directory name for each test method. + *******************************************************************************/ + public static void increaseTestInstanceCounter() + { + testInstanceCounter++; + } + + + + /******************************************************************************* + ** Reset the counter to 0 (e.g., to let some tests have a known value). + *******************************************************************************/ + public static void resetTestInstanceCounter() + { + testInstanceCounter = 0; + } + + + + /******************************************************************************* + ** Get the current value of the testInstanceCounter. See {@link #increaseTestInstanceCounter()} + *******************************************************************************/ + public static int getTestInstanceCounter() + { + return (testInstanceCounter); + } + + + + /******************************************************************************* + ** Meant to run both after and before test methods - makes sure the file system + ** is empty for the path under the instance. + *******************************************************************************/ + public static void cleanInstanceFiles() throws IOException + { + FileUtils.deleteDirectory(new File(BASE_PATH + File.separator + testInstanceCounter)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QInstance defineInstance() throws QInstanceValidationException + { + QInstance qInstance = new QInstance(); + qInstance.setAuthentication(defineAuthentication()); + qInstance.addBackend(defineLocalFilesystemBackend()); + qInstance.addTable(defineLocalFilesystemJSONPersonTable()); + qInstance.addBackend(defineS3Backend()); + qInstance.addTable(defineS3CSVPersonTable()); + + new QInstanceValidator().validate(qInstance); + + return (qInstance); + } + + + + /******************************************************************************* + ** Define the authentication used in standard tests - using 'mock' type. + ** + *******************************************************************************/ + public static QAuthenticationMetaData defineAuthentication() + { + return new QAuthenticationMetaData() + .withName("mock") + .withType("mock"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static FilesystemBackendMetaData defineLocalFilesystemBackend() + { + return (new FilesystemBackendMetaData() + .withBasePath(BASE_PATH + File.separator + testInstanceCounter) + .withName(BACKEND_NAME_LOCAL_FS)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineLocalFilesystemJSONPersonTable() + { + return new QTableMetaData() + .withName(TABLE_NAME_PERSON_LOCAL_FS) + .withLabel("Person") + .withBackendName(defineLocalFilesystemBackend().getName()) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date")) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) + .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name")) + .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name")) + .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) + .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withBackendDetails(new FilesystemTableBackendDetails() + .withBasePath("persons") + .withRecordFormat(RecordFormat.JSON) + .withCardinality(Cardinality.MANY) + ); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static S3BackendMetaData defineS3Backend() + { + return (new S3BackendMetaData() + .withBucketName(BaseS3Test.BUCKET_NAME) + .withBasePath(BaseS3Test.TEST_FOLDER) + .withName(BACKEND_NAME_S3)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineS3CSVPersonTable() + { + return new QTableMetaData() + .withName(TABLE_NAME_PERSON_S3) + .withLabel("Person S3 Table") + .withBackendName(defineS3Backend().getName()) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date")) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) + .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name")) + .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name")) + .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) + .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withBackendDetails(new S3TableBackendDetails() + .withRecordFormat(RecordFormat.CSV) + .withCardinality(Cardinality.MANY) + ); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QSession getMockSession() + { + MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule(); + return (mockAuthenticationModule.createSession(null)); + } +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java new file mode 100644 index 00000000..9d172573 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java @@ -0,0 +1,175 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local; + + +import java.io.File; +import java.io.IOException; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for FilesystemBackendModule + *******************************************************************************/ +public class FilesystemBackendModuleTest +{ + private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist"; + + + + @BeforeEach + public void beforeEach() throws IOException + { + new FilesystemActionTest().primeFilesystem(); + } + + + + @AfterEach + public void afterEach() throws Exception + { + new FilesystemActionTest().cleanFilesystem(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteFile() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // first list the files - then delete one, then re-list, and assert that we have one fewer // + ///////////////////////////////////////////////////////////////////////////////////////////// + List filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + + FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); + filesystemBackendModule.getActionBase().deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath()); + + List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + Assertions.assertEquals(filesBeforeDelete.size() - 1, filesAfterDelete.size(), + "Should be one fewer file listed after deleting one."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteFileDoesNotExist() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count // + // note, our contract is that as long as the file doesn't exist after calling delete (e.g., if it wasn't there // + // to begin with, then we're okay, and don't expect an exception // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + + FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); + filesystemBackendModule.getActionBase().deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + + List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + Assertions.assertEquals(filesBeforeDelete.size(), filesAfterDelete.size(), + "Should be same number of files after deleting bogus path"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMoveFile() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath(); + String subPath = basePath + File.separator + "subdir"; + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // first list the files (non-recursively) - then move one into a sub-folder, then re-list, and // + // assert that we have one fewer then list again including sub-folders, and see the changed count // + //////////////////////////////////////////////////////////////////////////////////////////////////// + List filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + + FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); + String originalFilePath = filesBeforeMove.get(0).getAbsolutePath(); + String movedFilePath = originalFilePath.replace(basePath, subPath); + + filesystemBackendModule.getActionBase().moveFile(qInstance, table, originalFilePath, movedFilePath); + + List filesAfterMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + Assertions.assertEquals(filesBeforeMove.size() - 1, filesAfterMove.size(), "Should be one fewer file in the listing after moving one."); + + ////////////////////////////////////////////////////////////////////////// + // move the file back and assert that the count goes back to the before // + ////////////////////////////////////////////////////////////////////////// + filesystemBackendModule.getActionBase().moveFile(qInstance, table, movedFilePath, originalFilePath); + + List filesAfterMoveBack = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + Assertions.assertEquals(filesBeforeMove.size(), filesAfterMoveBack.size(), "Should be original number of files after moving back"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMoveFileDoesNotExit() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath(); + String subPath = basePath + File.separator + "subdir"; + List filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + String originalFilePath = filesBeforeMove.get(0).getAbsolutePath(); + String movedFilePath = originalFilePath.replace(basePath, subPath); + + Assertions.assertThrows(FilesystemException.class, () -> + { + FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); + filesystemBackendModule.getActionBase().moveFile(qInstance, table, PATH_THAT_WONT_EXIST, movedFilePath); + }); + + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java new file mode 100644 index 00000000..755ab477 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java @@ -0,0 +1,128 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import java.io.File; +import java.io.IOException; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** Base class for Filesystem action tests. + ** + ** Knows how to set up the filesystem for the tests. + *******************************************************************************/ +public class FilesystemActionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + public void beforeEach() throws Exception + { + primeFilesystem(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + public void afterEach() throws Exception + { + cleanFilesystem(); + } + + + + /******************************************************************************* + ** Set up the file system + *******************************************************************************/ + public void primeFilesystem() throws IOException + { + TestUtils.cleanInstanceFiles(); + TestUtils.increaseTestInstanceCounter(); + FilesystemBackendMetaData filesystemBackendMetaData = TestUtils.defineLocalFilesystemBackend(); + + File baseDirectory = new File(filesystemBackendMetaData.getBasePath()); + boolean mkdirsResult = baseDirectory.mkdirs(); + if(!mkdirsResult) + { + fail("Failed to make directories at [" + baseDirectory + "] for filesystem backend module"); + } + + writePersonFiles(baseDirectory); + } + + + + /******************************************************************************* + ** Write some data files into the directory for the filesystem module. + *******************************************************************************/ + private void writePersonFiles(File baseDirectory) throws IOException + { + String fullPath = baseDirectory.getAbsolutePath(); + if (TestUtils.defineLocalFilesystemJSONPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details) + { + if (StringUtils.hasContent(details.getBasePath())) + { + fullPath += File.separatorChar + details.getBasePath(); + } + } + fullPath += File.separatorChar; + + String jsonData1 = """ + [ + {"id":1,"createDate":"2021-10-26 14:39:37","modifyDate":"2021-10-26 14:39:37","firstName":"John","lastName":"Doe","birthDate":"1981-01-01","email":"john@kingsrook.com"}, + {"id":2,"createDate":"2022-06-17 14:52:59","modifyDate":"2022-06-17 14:52:59","firstName":"Jane","lastName":"Smith","birthDate":"1982-02-02","email":"jane@kingsrook.com"} + ] + """; + FileUtils.writeStringToFile(new File(fullPath + "DATA-1.json"), jsonData1); + + String jsonData2 = """ + [ + {"id":3,"createDate":"2021-11-27 15:40:38","modifyDate":"2021-11-27 15:40:38","firstName":"Homer","lastName":"S","birthDate":"1983-03-03","email":"homer.s@kingsrook.com"} + ] + """; + FileUtils.writeStringToFile(new File(fullPath + "DATA-2.json"), jsonData2); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void cleanFilesystem() throws IOException + { + TestUtils.cleanInstanceFiles(); + } +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java new file mode 100644 index 00000000..de3e21d8 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java @@ -0,0 +1,68 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilesystemQueryActionTest extends FilesystemActionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQuery1() throws QException + { + QueryRequest queryRequest = initQueryRequest(); + QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest); + Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows"); + Assertions.assertTrue(queryResult.getRecords().stream() + .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)), + "All records should have a full-path in their backend details, matching the test folder name"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QueryRequest initQueryRequest() throws QInstanceValidationException + { + QueryRequest queryRequest = new QueryRequest(); + queryRequest.setInstance(TestUtils.defineInstance()); + queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName()); + return queryRequest; + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java new file mode 100644 index 00000000..7c6515f6 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata; + + +import java.io.IOException; +import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for FilesystemBackendMetaData + *******************************************************************************/ +class FilesystemBackendMetaDataTest +{ + + + /******************************************************************************* + ** Test that an instance can be serialized as expected + *******************************************************************************/ + @Test + public void testSerializingToJson() throws QInstanceValidationException + { + TestUtils.resetTestInstanceCounter(); + QInstance qInstance = TestUtils.defineInstance(); + String json = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance); + System.out.println(JsonUtils.prettyPrint(json)); + System.out.println(json); + String expectToContain = """ + "local-filesystem":{"basePath":"/tmp/filesystem-tests/0","backendType":"filesystem","name":"local-filesystem"}"""; + assertTrue(json.contains(expectToContain)); + } + + + + /******************************************************************************* + ** Test that an instance can be deserialized as expected + *******************************************************************************/ + @Test + public void testDeserializingFromJson() throws IOException, QInstanceValidationException + { + QInstanceAdapter qInstanceAdapter = new QInstanceAdapter(); + + QInstance qInstance = TestUtils.defineInstance(); + String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance); + + QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json); + assertThat(deserialized).usingRecursiveComparison() + .ignoringFields("hasBeenValidated") // note, this field is @JsonIgnore + .isEqualTo(qInstance); + } +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java new file mode 100644 index 00000000..8da9dbda --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java @@ -0,0 +1,228 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic; + + +import java.io.File; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for BasicETLCleanupSourceFilesFunction + *******************************************************************************/ +public class BasicETLCleanupSourceFilesFunctionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDelete1Record1File() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath = getRandomFilePathPersonTable(qInstance); + testDelete(qInstance, List.of(filePath)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDelete2Records1File() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath = getRandomFilePathPersonTable(qInstance); + testDelete(qInstance, List.of(filePath, filePath)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDelete2Record2File() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath1 = getRandomFilePathPersonTable(qInstance); + String filePath2 = getRandomFilePathPersonTable(qInstance); + testDelete(qInstance, List.of(filePath1, filePath2)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMove1Record1File() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath = getRandomFilePathPersonTable(qInstance); + testMove(qInstance, List.of(filePath)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMove2Records1File() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath = getRandomFilePathPersonTable(qInstance); + testMove(qInstance, List.of(filePath, filePath)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMove2Record2File() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath1 = getRandomFilePathPersonTable(qInstance); + String filePath2 = getRandomFilePathPersonTable(qInstance); + testMove(qInstance, List.of(filePath1, filePath2)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void testDelete(QInstance qInstance, List filePaths) throws Exception + { + RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of( + BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_DELETE, + // todo - even though this field isn't needed, since we gave a value of "delete" + // the RunFunctionAction considers any missing input to be an error... + BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, "")); + + assertNull(runFunctionResult.getError()); + for(String filePath : filePaths) + { + assertFalse(new File(filePath).exists(), "File should have been deleted."); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void testMove(QInstance qInstance, List filePaths) throws Exception + { + String trashDir = File.separator + "tmp" + File.separator + "trash"; + RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of( + BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_MOVE, + BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir)); + + assertNull(runFunctionResult.getError()); + + for(String filePath : filePaths) + { + assertFalse(new File(filePath).exists(), "File should have been moved."); + + String movedPath = trashDir + File.separator + (new File(filePath).getName()); + assertTrue(new File(movedPath).exists(), "File should have been moved."); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private RunFunctionResult runFunction(QInstance qInstance, List filePaths, Map values) throws Exception + { + QFunctionMetaData qFunctionMetaData = new BasicETLCleanupSourceFilesFunction().defineFunctionMetaData(); + QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addFunction(qFunctionMetaData); + qInstance.addProcess(qProcessMetaData); + + HashSet filePathsSet = new HashSet<>(filePaths); + for(String filePath : filePathsSet) + { + File file = new File(filePath); + FileUtils.writeStringToFile(file, "content"); + } + + // List records = filePaths.stream() + // .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList(); + + RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); + runFunctionRequest.setFunctionName(qFunctionMetaData.getName()); + runFunctionRequest.setProcessName(qProcessMetaData.getName()); + // runFunctionRequest.setRecords(records); + runFunctionRequest.setSession(TestUtils.getMockSession()); + runFunctionRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + runFunctionRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3); + runFunctionRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet)); + + for(Map.Entry entry : values.entrySet()) + { + runFunctionRequest.addValue(entry.getKey(), entry.getValue()); + } + + RunFunctionAction runFunctionAction = new RunFunctionAction(); + return (runFunctionAction.execute(runFunctionRequest)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getRandomFilePathPersonTable(QInstance qInstance) + { + FilesystemBackendMetaData backend = (FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS); + FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS).getBackendDetails(); + String tablePath = backend.getBasePath() + File.separator + backendDetails.getBasePath(); + String filePath = tablePath + File.separator + UUID.randomUUID(); + return filePath; + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java new file mode 100644 index 00000000..2064617c --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java @@ -0,0 +1,266 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync; + + +import java.util.List; +import java.util.stream.Collectors; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; +import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; +import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModuleSubclassForTest; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for FilesystemSyncProcess using S3 backend + *******************************************************************************/ +class FilesystemSyncProcessS3Test extends BaseS3Test +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws Exception + { + QBackendModuleDispatcher.registerBackendModule(new S3BackendModuleSubclassForTest()); + QInstance qInstance = TestUtils.defineInstance(); + + String sourceBucket = "source-bucket"; + String archiveBucket = "archive-bucket"; + String processingBucket = "processing-bucket"; + + getAmazonS3().createBucket(sourceBucket); + getAmazonS3().createBucket(archiveBucket); + getAmazonS3().createBucket(processingBucket); + + S3BackendMetaData sourceBackend = defineBackend(qInstance, "source", sourceBucket); + S3BackendMetaData archiveBackend = defineBackend(qInstance, "archive", archiveBucket); + S3BackendMetaData processingBackend = defineBackend(qInstance, "processing", processingBucket); + + QTableMetaData sourceTable = defineTable(qInstance, "source", sourceBackend, "source", "*/l3/*.csv"); + QTableMetaData archiveTable = defineTable(qInstance, "archive", archiveBackend, "archive", "*/l3/*.csv"); + QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv"); + + QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); + QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); + qInstance.addProcess(process); + + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); + + /////////////////////////// + // write some test files // + /////////////////////////// + writeTestFile(sourceBackend, sourceTable, "foo/l3/1.csv", "x"); + writeTestFile(sourceBackend, sourceTable, "bar/l3/2.csv", "x"); + writeTestFile(archiveBackend, archiveTable, "foo/l3/1.csv", "x"); + + printTableListing(sourceBackend, sourceTable); + printTableListing(archiveBackend, archiveTable); + printTableListing(processingBackend, processingTable); + + ////////////////////// + // run the function // + ////////////////////// + RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); + runFunctionRequest.setFunctionName(function.getName()); + runFunctionRequest.setProcessName(process.getName()); + runFunctionRequest.setSession(TestUtils.getMockSession()); + + RunFunctionAction runFunctionAction = new RunFunctionAction(); + RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); + System.out.println(runFunctionResult); + + printTableListing(sourceBackend, sourceTable); + printTableListing(archiveBackend, archiveTable); + printTableListing(processingBackend, processingTable); + + assertTableListing(archiveBackend, archiveTable, "root/archive/foo/l3/1.csv", "root/archive/bar/l3/2.csv"); + assertTableListing(processingBackend, processingTable, "root/processing/bar/l3/2.csv"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testArchiveAndProcessingInSameBucket() throws Exception + { + QBackendModuleDispatcher.registerBackendModule(new S3BackendModuleSubclassForTest()); + QInstance qInstance = TestUtils.defineInstance(); + + String vendorBucket = "vendor-bucket"; + String localBucket = "local-bucket"; + + getAmazonS3().createBucket(vendorBucket); + getAmazonS3().createBucket(localBucket); + + S3BackendMetaData vendorBackend = defineBackend(qInstance, "source", vendorBucket); + S3BackendMetaData localBackend = defineBackend(qInstance, "archive", localBucket); + + QTableMetaData sourceTable = defineTable(qInstance, "source", vendorBackend, "source", "*/l3/*.csv"); + QTableMetaData archiveTable = defineTable(qInstance, "archive", localBackend, "archive", "*/l3/*.csv"); + QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv"); + + QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); + QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); + qInstance.addProcess(process); + + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); + + /////////////////////////// + // write some test files // + /////////////////////////// + writeTestFile(vendorBackend, sourceTable, "foo/l3/1.csv", "x"); + writeTestFile(vendorBackend, sourceTable, "bar/l3/2.csv", "x"); + writeTestFile(localBackend, archiveTable, "foo/l3/1.csv", "x"); + + printTableListing(vendorBackend, sourceTable); + printTableListing(localBackend, archiveTable); + printTableListing(localBackend, processingTable); + + ////////////////////// + // run the function // + ////////////////////// + RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); + runFunctionRequest.setFunctionName(function.getName()); + runFunctionRequest.setProcessName(process.getName()); + runFunctionRequest.setSession(TestUtils.getMockSession()); + + RunFunctionAction runFunctionAction = new RunFunctionAction(); + RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); + System.out.println(runFunctionResult); + + printTableListing(vendorBackend, sourceTable); + printTableListing(localBackend, archiveTable); + printTableListing(localBackend, processingTable); + + assertTableListing(localBackend, archiveTable, "root/archive/foo/l3/1.csv", "root/archive/bar/l3/2.csv"); + assertTableListing(localBackend, processingTable, "root/processing/bar/l3/2.csv"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException + { + S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend); + AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase(); + + List s3ObjectSummaries = actionBase.listFiles(table, backend); + assertEquals(paths.length, s3ObjectSummaries.size(), "Expected number of files in table: " + table.getName()); + for(String path : paths) + { + assertTrue(s3ObjectSummaries.stream().anyMatch(s3o -> s3o.getKey().equals(path)), + "Path [" + path + "] should be in the listing, but was not. Full listing is: " + + s3ObjectSummaries.stream().map(S3ObjectSummary::getKey).collect(Collectors.joining(","))); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException + { + S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend); + AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase(); + + System.out.println("Files in: " + table.getName()); + actionBase.listFiles(table, backend).forEach(o -> System.out.println(o.getKey())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeTestFile(S3BackendMetaData backend, QTableMetaData table, String name, String content) throws Exception + { + S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend); + AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase(); + String fullPath = actionBase.getFullBasePath(table, backend); + + actionBase.writeFile(backend, fullPath + "/" + name, content.getBytes()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private S3BackendMetaData defineBackend(QInstance qInstance, String which, String bucketName) + { + QBackendMetaData backendMetaData = new S3BackendMetaData() + .withBucketName(bucketName) + .withBasePath("root") + .withName("backend-" + which); + qInstance.addBackend(backendMetaData); + return (S3BackendMetaData) backendMetaData; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineTable(QInstance qInstance, String which, QBackendMetaData backend, String path, String glob) + { + QTableMetaData qTableMetaData = new QTableMetaData() + .withName("table-" + which) + .withBackendName(backend.getName()) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withBackendDetails(new S3TableBackendDetails() + .withBasePath(path) + .withGlob(glob)); + qInstance.addTable(qTableMetaData); + return (qTableMetaData); + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java new file mode 100644 index 00000000..188ef71a --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java @@ -0,0 +1,123 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync; + + +import java.io.File; +import java.io.IOException; +import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for FilesystemSyncProcess + *******************************************************************************/ +class FilesystemSyncProcessTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws Exception + { + TestUtils.cleanInstanceFiles(); + + QTableMetaData sourceTable = defineTable("source"); + QTableMetaData archiveTable = defineTable("archive"); + QTableMetaData processingTable = defineTable("processing"); + QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); + QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); + + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); + // function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1); + + QInstance qInstance = TestUtils.defineInstance(); + qInstance.addTable(sourceTable); + qInstance.addTable(archiveTable); + qInstance.addTable(processingTable); + qInstance.addProcess(process); + + /////////////////////////// + // write some test files // + /////////////////////////// + String basePath = ((FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS)).getBasePath(); + writeTestFile(basePath, sourceTable, "1.txt", "x"); + writeTestFile(basePath, sourceTable, "2.txt", "x"); + // writeTestFile(basePath, sourceTable, "3.txt", "x"); + writeTestFile(basePath, archiveTable, "2.txt", "x"); + + ////////////////////// + // run the function // + ////////////////////// + RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); + runFunctionRequest.setFunctionName(function.getName()); + runFunctionRequest.setProcessName(process.getName()); + runFunctionRequest.setSession(TestUtils.getMockSession()); + + RunFunctionAction runFunctionAction = new RunFunctionAction(); + RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); + System.out.println(runFunctionResult); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeTestFile(String basePath, QTableMetaData table, String name, String content) throws IOException + { + String path = ((FilesystemTableBackendDetails) table.getBackendDetails()).getBasePath(); + File file = new File(basePath + "/" + path + "/" + name); + FileUtils.writeStringToFile(file, content); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineTable(String subPath) + { + return new QTableMetaData() + .withName("table-" + subPath) + .withBackendName(TestUtils.BACKEND_NAME_LOCAL_FS) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withBackendDetails(new FilesystemTableBackendDetails() + .withBasePath(subPath)); + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java new file mode 100644 index 00000000..490a10b7 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java @@ -0,0 +1,162 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3; + + +import cloud.localstack.ServiceName; +import cloud.localstack.awssdkv1.TestUtils; +import cloud.localstack.docker.LocalstackDockerExtension; +import cloud.localstack.docker.annotation.LocalstackDockerProperties; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; + + +/******************************************************************************* + ** Base class for tests that want to be able to work with localstack s3. + *******************************************************************************/ +@ExtendWith(LocalstackDockerExtension.class) +@LocalstackDockerProperties(services = { ServiceName.S3 }, portEdge = "2960", portElasticSearch = "2961") +public class BaseS3Test +{ + public static final String BUCKET_NAME = "localstack-test-bucket"; + public static final String TEST_FOLDER = "test-files"; + public static final String SUB_FOLDER = "sub-folder"; + + + + /******************************************************************************* + ** Before each unit test, get the test bucket into a known state + *******************************************************************************/ + @BeforeEach + public void beforeEach() + { + AmazonS3 amazonS3 = getAmazonS3(); + + amazonS3.createBucket(BUCKET_NAME); + amazonS3.putObject(BUCKET_NAME, "0.csv", getCSVHeader()); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/1.csv", getCSVData1()); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2()); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test"); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3()); + } + + + + /******************************************************************************* + ** After each unit test, clean up the bucket + *******************************************************************************/ + @AfterEach + public void afterEach() + { + AmazonS3 amazonS3 = getAmazonS3(); + + if(amazonS3.doesBucketExistV2(BUCKET_NAME)) + { + //////////////////////// + // todo - paginate... // + //////////////////////// + for(S3ObjectSummary objectSummary : amazonS3.listObjectsV2(BUCKET_NAME).getObjectSummaries()) + { + amazonS3.deleteObject(BUCKET_NAME, objectSummary.getKey()); + } + amazonS3.deleteBucket(BUCKET_NAME); + } + } + + + + /******************************************************************************* + ** Access a localstack-configured s3 client. + *******************************************************************************/ + protected AmazonS3 getAmazonS3() + { + return (TestUtils.getClientS3()); + } + + + + /******************************************************************************* + ** Access the S3Utils object, with localstack-configured s3 client. + *******************************************************************************/ + protected S3Utils getS3Utils() + { + S3Utils s3Utils = new S3Utils(); + s3Utils.setAmazonS3(getAmazonS3()); + return (s3Utils); + } + + + + /******************************************************************************* + ** Access a string of CSV test data. + *******************************************************************************/ + protected String getCSVHeader() + { + return (""" + "id","createDate","modifyDate","firstName","lastName","birthDate","email" + """); + } + + + + /******************************************************************************* + ** Access a string of CSV test data. + *******************************************************************************/ + protected String getCSVData1() + { + return (getCSVHeader() + """ + "1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com" + "2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com" + """); + } + + + + /******************************************************************************* + ** Access a string of CSV test data. + *******************************************************************************/ + protected String getCSVData2() + { + return (getCSVHeader() + """ + "3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com" + "4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com" + "5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com" + """); + } + + + + /******************************************************************************* + ** Access a string of CSV test data. + *******************************************************************************/ + protected String getCSVData3() + { + return (getCSVHeader() + """ + "6","2022-06-20 15:31:02","2022-06-20 15:31:02","Lisa","S","1986-06-06","lisa.s@kingsrook.com" + """); + } + +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleSubclassForTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleSubclassForTest.java new file mode 100644 index 00000000..84e5189f --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleSubclassForTest.java @@ -0,0 +1,53 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3; + + +import com.amazonaws.services.s3.model.S3ObjectSummary; +import cloud.localstack.awssdkv1.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; +import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; + + +/******************************************************************************* + ** Subclass of the S3Backend module, meant for use in unit tests, if/where we + ** need to make sure we use the localstack version of the S3 client. + *******************************************************************************/ +public class S3BackendModuleSubclassForTest extends S3BackendModule +{ + + /******************************************************************************* + ** Seed the AbstractS3Action with an s3Utils object that has the localstack + ** s3 client in it + *******************************************************************************/ + @Override + public AbstractBaseFilesystemAction getActionBase() + { + AbstractS3Action actionBase = (AbstractS3Action) super.getActionBase(); + S3Utils s3Utils = new S3Utils(); + s3Utils.setAmazonS3(TestUtils.getClientS3()); + actionBase.setS3Utils(s3Utils); + return (actionBase); + } + +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java new file mode 100644 index 00000000..44d1796d --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java @@ -0,0 +1,156 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3; + + +import java.util.List; +import java.util.UUID; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for S3BackendModule + *******************************************************************************/ +public class S3BackendModuleTest extends BaseS3Test +{ + private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteFile() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // first list the files - then delete one, then re-list, and assert that we have one fewer // + ///////////////////////////////////////////////////////////////////////////////////////////// + List s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); + + S3BackendModule s3BackendModule = new S3BackendModule(); + AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); + actionBase.setS3Utils(getS3Utils()); + actionBase.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey()); + + List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); + Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(), + "Should be one fewer file listed after deleting one."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteFileDoesNotExist() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count // + // note, we'd like to detect the non-delete, but there's no such info back from aws it appears? // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); + + S3BackendModule s3BackendModule = new S3BackendModule(); + AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); + actionBase.setS3Utils(getS3Utils()); + actionBase.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + + List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); + Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size(), s3ObjectSummariesAfterDelete.size(), + "Should be same number of files after deleting bogus path"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMoveFile() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3); + String subPath = TEST_FOLDER + "/" + SUB_FOLDER; + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // first list the files (non-recursively) - then move one into a sub-folder, then re-list, and // + // assert that we have one fewer then list again including sub-folders, and see the changed count // + //////////////////////////////////////////////////////////////////////////////////////////////////// + List s3ObjectSummariesBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); + List s3ObjectSummariesInSubFolderBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, subPath, ""); + List s3ObjectSummariesRecursiveBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**"); + + S3BackendModule s3BackendModule = new S3BackendModule(); + AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); + actionBase.setS3Utils(getS3Utils()); + String key = s3ObjectSummariesBeforeMove.get(0).getKey(); + actionBase.moveFile(qInstance, table, key, key.replaceFirst(TEST_FOLDER, subPath)); + + List s3ObjectSummariesAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); + List s3ObjectSummariesRecursiveAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**"); + List s3ObjectSummariesInSubFolderAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, subPath, ""); + + Assertions.assertEquals(s3ObjectSummariesBeforeMove.size() - 1, s3ObjectSummariesAfterMove.size(), + "Should be one fewer file in the non-recursive listing after moving one."); + Assertions.assertEquals(s3ObjectSummariesRecursiveBeforeMove.size(), s3ObjectSummariesRecursiveAfterMove.size(), + "Should be same number of files in the recursive listing before and after the move"); + Assertions.assertEquals(s3ObjectSummariesInSubFolderBeforeMove.size() + 1, s3ObjectSummariesInSubFolderAfterMove.size(), + "Should be one move file in the sub-folder listing after moving one."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMoveFileDoesNotExit() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3); + String subPath = TEST_FOLDER + "/" + SUB_FOLDER; + + S3BackendModule s3BackendModule = new S3BackendModule(); + AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); + actionBase.setS3Utils(getS3Utils()); + + Assertions.assertThrows(FilesystemException.class, () -> + actionBase.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, subPath + "/" + UUID.randomUUID()) + ); + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java new file mode 100644 index 00000000..ae490a44 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java @@ -0,0 +1,71 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3QueryActionTest extends BaseS3Test +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQuery1() throws QException + { + QueryRequest queryRequest = initQueryRequest(); + S3QueryAction s3QueryAction = new S3QueryAction(); + s3QueryAction.setS3Utils(getS3Utils()); + QueryResult queryResult = s3QueryAction.execute(queryRequest); + Assertions.assertEquals(5, queryResult.getRecords().size(), "Expected # of rows from unfiltered query"); + Assertions.assertTrue(queryResult.getRecords().stream() + .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseS3Test.TEST_FOLDER)), + "All records should have a full-path in their backend details, matching the test folder name"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QueryRequest initQueryRequest() throws QInstanceValidationException + { + QueryRequest queryRequest = new QueryRequest(); + queryRequest.setInstance(TestUtils.defineInstance()); + queryRequest.setTableName(TestUtils.defineS3CSVPersonTable().getName()); + return queryRequest; + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java new file mode 100644 index 00000000..acef8ec6 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata; + + +import java.io.IOException; +import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for S3BackendMetaData + *******************************************************************************/ +class S3BackendMetaDataTest +{ + + + /******************************************************************************* + ** Test that an instance can be serialized as expected + *******************************************************************************/ + @Test + public void testSerializingToJson() throws QInstanceValidationException + { + TestUtils.resetTestInstanceCounter(); + QInstance qInstance = TestUtils.defineInstance(); + String json = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance); + System.out.println(JsonUtils.prettyPrint(json)); + System.out.println(json); + String expectToContain = """ + {"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","secretKey":null,"accessKey":null,"backendType":"s3","name":"s3","region":null}"""; + assertTrue(json.contains(expectToContain)); + } + + + + /******************************************************************************* + ** Test that an instance can be deserialized as expected + *******************************************************************************/ + @Test + public void testDeserializingFromJson() throws IOException, QInstanceValidationException + { + QInstanceAdapter qInstanceAdapter = new QInstanceAdapter(); + + QInstance qInstance = TestUtils.defineInstance(); + String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance); + + QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json); + assertThat(deserialized).usingRecursiveComparison() + .ignoringFields("hasBeenValidated") // note, this field is @JsonIgnore + .isEqualTo(qInstance); + } +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/PathMatcherGlobTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/PathMatcherGlobTest.java new file mode 100644 index 00000000..2aa65293 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/PathMatcherGlobTest.java @@ -0,0 +1,73 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.utils; + + +import java.net.URI; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Verification for some of the behavior in the S3Utils - working with PathMatcher + ** globs. + *******************************************************************************/ +public class PathMatcherGlobTest +{ + + @Test + public void testPathMatcher() throws Exception + { + ///////////////////////////////////////////////////////////// + // note: must start with for both the pattern and the uri // + ///////////////////////////////////////////////////////////// + PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/*/acme/*/*.csv"); + + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/1234.csv"))), "Glob should match"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/nj/acme/20220627/1234.csv"))), "Glob should match"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220628/1234.csv"))), "Glob should match"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/12345.csv"))), "Glob should match"); + + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/beta/20220627/1234.csv"))), "Glob should not match (beta vs acme)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///something/stl/acme/20220627/1234.csv"))), "Glob should not match (wrong start path)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/csv"))), "Glob should not match (no file basename)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/1234.CSV"))), "Glob should not match (wrong case extension)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/extra/1234.csv"))), "Glob should not match (extra dir)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/extra/acme/20220627/1234.csv"))), "Glob should not match (extra dir)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/extra/stl/acme/20220627/1234.csv"))), "Glob should not match (extra dir)"); + + pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/**/acme/*/*.csv"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/extra/stl/acme/20220627/1234.csv"))), "Glob should match with extra dir"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/extra/extra2/stl/acme/20220627/1234.csv"))), "Glob should match with 2 extra dirs"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/acme/20220627/1234.csv"))), "Glob does not match with no dir for **"); + + pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/**"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/1234.csv"))), "Glob should match with extra dir"); + + pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/*"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///1234.csv"))), "Glob should match"); + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java new file mode 100644 index 00000000..ed4afbdf --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java @@ -0,0 +1,84 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.utils; + + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3UtilsTest extends BaseS3Test +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testListObjectsInBucketAtPath() + { + S3Utils s3Utils = getS3Utils(); + assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders"); + assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders"); + assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders"); + assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders"); + assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash"); + assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash"); + assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, "").size(), "Just in the subfolder non-recursive"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, "/**").size(), "Just in the subfolder recursive"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "//" + SUB_FOLDER, "/**").size(), "Just in the subfolder recursive, multi /"); + assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "not-a-real-path/", "").size(), "In a non-existing folder"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string"); + assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively"); + assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testGetObjectAsInputStream() throws IOException + { + S3Utils s3Utils = getS3Utils(); + List s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", ""); + S3ObjectSummary s3ObjectSummary = s3ObjectSummaries.stream().filter(o -> o.getKey().contains("1.csv")).findAny().get(); + InputStream inputStream = s3Utils.getObjectAsInputStream(s3ObjectSummary); + String csvFromS3 = IOUtils.toString(inputStream); + + assertEquals(getCSVData1(), csvFromS3, "File from S3 should match expected content"); + } + +} \ No newline at end of file