diff --git a/pom.xml b/pom.xml index 2d069357..0c467092 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ com.kingsrook.qqq qqq-backend-core - 0.0.0-20220623.214704-10 + 0.0.0-20220628.161829-14 diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java index 9a7578a4..3051e18a 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java @@ -22,30 +22,18 @@ package com.kingsrook.qqq.backend.module.filesystem.base; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; -import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; /******************************************************************************* ** Interface to add additional functionality commonly among the various filesystem ** module implementations. *******************************************************************************/ -public interface FilesystemBackendModuleInterface +public interface FilesystemBackendModuleInterface { /******************************************************************************* - ** In contrast with the DeleteAction, which deletes RECORDS - this is a - ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE - ** e.g., for post-ETL. - ** - ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + ** For filesystem backends, get the module-specific action base-class, that helps + ** with functions like listing and deleting files. *******************************************************************************/ - void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException; - - /******************************************************************************* - ** Move a file from a source path, to a destination path. - ** - ** @throws FilesystemException if the move is known to have failed - *******************************************************************************/ - void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException; + AbstractBaseFilesystemAction getActionBase(); } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index b8e07308..1df77c93 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -27,27 +27,42 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import java.util.function.Function; import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractQTableRequest; import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.NotImplementedException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* ** Base class for all Filesystem actions across all modules. + ** + ** @param FILE The class that represents a file in the sub-module. e.g., + * a java.io.File, or an S3Object. *******************************************************************************/ public abstract class AbstractBaseFilesystemAction { + private static final Logger LOG = LogManager.getLogger(AbstractBaseFilesystemAction.class); + + /******************************************************************************* ** List the files for a table - to be implemented in module-specific subclasses. @@ -60,31 +75,79 @@ public abstract class AbstractBaseFilesystemAction public abstract InputStream readFile(FILE file) throws IOException; /******************************************************************************* - ** Add backend details to records about the file that they are in. + ** Write a file - to be implemented in module-specific subclasses. *******************************************************************************/ - protected abstract void addBackendDetailsToRecords(List recordsInFile, FILE file); + public abstract void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException; + + /******************************************************************************* + ** Get a string that represents the full path to a file. + *******************************************************************************/ + protected abstract String getFullPathForFile(FILE file); + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + *******************************************************************************/ + public abstract void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException; + + /******************************************************************************* + ** Move a file from a source path, to a destination path. + ** + ** @throws FilesystemException if the move is known to have failed + *******************************************************************************/ + public abstract void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException; + + /******************************************************************************* + ** e.g., with a base path of /foo/ + ** and a table path of /bar/ + ** and a file at /foo/bar/baz.txt + ** give us just the baz.txt part. + *******************************************************************************/ + public abstract String stripBackendAndTableBasePathsFromFileName(FILE file, QBackendMetaData sourceBackend, QTableMetaData sourceTable); + /******************************************************************************* - ** Append together the backend's base path (if present), with a table's path (again, if present). + ** Append together the backend's base path (if present), with a table's base + ** path (again, if present). *******************************************************************************/ - protected String getFullPath(QTableMetaData table, QBackendMetaData backendBase) + public String getFullBasePath(QTableMetaData table, QBackendMetaData backendBase) { AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase); String fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : ""; AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); - if(StringUtils.hasContent(tableDetails.getPath())) + if(StringUtils.hasContent(tableDetails.getBasePath())) { - fullPath += File.separatorChar + tableDetails.getPath(); + fullPath += File.separatorChar + tableDetails.getBasePath(); } fullPath += File.separatorChar; + fullPath = stripDuplicatedSlashes(fullPath); + return fullPath; } + /******************************************************************************* + ** + *******************************************************************************/ + public static String stripDuplicatedSlashes(String path) + { + if(path == null) + { + return (null); + } + + return (path.replaceAll("//+", "/")); + } + + + /******************************************************************************* ** Get the backend metaData, type-checked as the requested type. *******************************************************************************/ @@ -119,6 +182,8 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ public QueryResult executeQuery(QueryRequest queryRequest) throws QException { + preAction(queryRequest); + try { QueryResult rs = new QueryResult(); @@ -131,20 +196,25 @@ public abstract class AbstractBaseFilesystemAction for(FILE file : files) { + LOG.info("Processing file: " + getFullPathForFile(file)); switch(tableDetails.getRecordFormat()) { - case "csv": + case CSV: { - String fileContents = IOUtils.toString(readFile(file)); + String fileContents = IOUtils.toString(readFile(file)); + fileContents = customizeFileContentsAfterReading(table, fileContents); + List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); addBackendDetailsToRecords(recordsInFile, file); records.addAll(recordsInFile); break; } - case "json": + case JSON: { - String fileContents = IOUtils.toString(readFile(file)); + String fileContents = IOUtils.toString(readFile(file)); + fileContents = customizeFileContentsAfterReading(table, fileContents); + List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); addBackendDetailsToRecords(recordsInFile, file); @@ -162,8 +232,64 @@ public abstract class AbstractBaseFilesystemAction } catch(Exception e) { - e.printStackTrace(); + LOG.warn("Error executing query", e); throw new QException("Error executing query", e); } } + + + + /******************************************************************************* + ** Add backend details to records about the file that they are in. + *******************************************************************************/ + protected void addBackendDetailsToRecords(List recordsInFile, FILE file) + { + recordsInFile.forEach(record -> + { + record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, getFullPathForFile(file)); + }); + } + + + + /******************************************************************************* + ** Method that subclasses can override to add pre-action things (e.g., setting up + ** s3 client). + *******************************************************************************/ + protected void preAction(AbstractQTableRequest tableRequest) + { + ///////////////////////////////////////////////////////////////////// + // noop in base class - subclasses can add functionality if needed // + ///////////////////////////////////////////////////////////////////// + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String customizeFileContentsAfterReading(QTableMetaData table, String fileContents) throws QException + { + Optional optionalCustomizer = table.getCustomizer("postFileRead"); + if(optionalCustomizer.isEmpty()) + { + return (fileContents); + } + QCodeReference customizer = optionalCustomizer.get(); + + try + { + Class customizerClass = Class.forName(customizer.getName()); + + @SuppressWarnings("unchecked") + Function function = (Function) customizerClass.getConstructor().newInstance(); + + return function.apply(fileContents); + } + catch(Exception e) + { + throw (new QException("Error customizing file contents", e)); + } + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java index ca5eae58..e98e4fad 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java @@ -30,42 +30,78 @@ import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; *******************************************************************************/ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails { - private String path; - private String recordFormat; // todo - enum? but hard w/ serialization? - private String cardinality; // todo - enum? + private String basePath; + private String glob; + private RecordFormat recordFormat; + private Cardinality cardinality; /******************************************************************************* - ** Getter for path + ** Getter for basePath ** *******************************************************************************/ - public String getPath() + public String getBasePath() { - return path; + return basePath; } /******************************************************************************* - ** Setter for path + ** Setter for basePath ** *******************************************************************************/ - public void setPath(String path) + public void setBasePath(String basePath) { - this.path = path; + this.basePath = basePath; } /******************************************************************************* - ** Fluent Setter for path + ** Fluent Setter for basePath ** *******************************************************************************/ @SuppressWarnings("unchecked") - public T withPath(String path) + public T withBasePath(String basePath) { - this.path = path; + this.basePath = basePath; + return (T) this; + } + + + + /******************************************************************************* + ** Getter for glob + ** + *******************************************************************************/ + public String getGlob() + { + return glob; + } + + + + /******************************************************************************* + ** Setter for glob + ** + *******************************************************************************/ + public void setGlob(String glob) + { + this.glob = glob; + } + + + + /******************************************************************************* + ** Fluent Setter for glob + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withGlob(String glob) + { + this.glob = glob; return (T) this; } @@ -75,7 +111,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails ** Getter for recordFormat ** *******************************************************************************/ - public String getRecordFormat() + public RecordFormat getRecordFormat() { return recordFormat; } @@ -86,7 +122,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails ** Setter for recordFormat ** *******************************************************************************/ - public void setRecordFormat(String recordFormat) + public void setRecordFormat(RecordFormat recordFormat) { this.recordFormat = recordFormat; } @@ -98,7 +134,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails ** *******************************************************************************/ @SuppressWarnings("unchecked") - public T withRecordFormat(String recordFormat) + public T withRecordFormat(RecordFormat recordFormat) { this.recordFormat = recordFormat; return ((T) this); @@ -110,7 +146,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails ** Getter for cardinality ** *******************************************************************************/ - public String getCardinality() + public Cardinality getCardinality() { return cardinality; } @@ -121,7 +157,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails ** Setter for cardinality ** *******************************************************************************/ - public void setCardinality(String cardinality) + public void setCardinality(Cardinality cardinality) { this.cardinality = cardinality; } @@ -133,7 +169,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails ** *******************************************************************************/ @SuppressWarnings("unchecked") - public T withCardinality(String cardinality) + public T withCardinality(Cardinality cardinality) { this.cardinality = cardinality; return ((T) this); diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/Cardinality.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/Cardinality.java new file mode 100644 index 00000000..0fae2ae6 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/Cardinality.java @@ -0,0 +1,33 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; + + +/******************************************************************************* + ** Mode for filesystem backends: are all records in a single file, or are there + ** many files? + *******************************************************************************/ +public enum Cardinality +{ + ONE, + MANY +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/RecordFormat.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/RecordFormat.java new file mode 100644 index 00000000..572876de --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/RecordFormat.java @@ -0,0 +1,34 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; + + +/******************************************************************************* + ** How are records stored in the files in a filesystem backend? CSV, JSON, + ** future types may XML, or more exotic ones, like "BINARY" or "TEXT" (e.g., 1 + ** record and 1 field per-file) + *******************************************************************************/ +public enum RecordFormat +{ + CSV, + JSON +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java index 09594b99..f432ec7b 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java @@ -24,16 +24,15 @@ package com.kingsrook.qqq.backend.module.filesystem.local; import java.io.File; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; -import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction; @@ -52,6 +51,19 @@ public class FilesystemBackendModule implements QBackendModuleInterface, Filesys private static final Logger LOG = LogManager.getLogger(FilesystemBackendModule.class); + + /******************************************************************************* + ** For filesystem backends, get the module-specific action base-class, that helps + ** with functions like listing and deleting files. + *******************************************************************************/ + @Override + public AbstractBaseFilesystemAction getActionBase() + { + return (new AbstractFilesystemAction()); + } + + + /******************************************************************************* ** Method where a backend module must be able to provide its type (name). *******************************************************************************/ @@ -127,70 +139,4 @@ public class FilesystemBackendModule implements QBackendModuleInterface, Filesys return (new FilesystemDeleteAction()); } - - - /******************************************************************************* - ** In contrast with the DeleteAction, which deletes RECORDS - this is a - ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE - ** e.g., for post-ETL. - ** - ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit - *******************************************************************************/ - @Override - public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException - { - File file = new File(fileReference); - if(!file.exists()) - { - ////////////////////////////////////////////////////////////////////////////////////////////// - // if the file doesn't exist, just exit with noop. don't throw an error - that should only // - // happen if the "contract" of the method is broken, and the file still exists // - ////////////////////////////////////////////////////////////////////////////////////////////// - LOG.debug("Not deleting file [{}], because it does not exist.", file); - return; - } - - if(!file.delete()) - { - throw (new FilesystemException("Failed to delete file: " + fileReference)); - } - } - - - - /******************************************************************************* - ** Move a file from a source path, to a destination path. - ** - ** @throws FilesystemException if the delete is known to have failed - *******************************************************************************/ - @Override - public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException - { - File sourceFile = new File(source); - File destinationFile = new File(destination); - File destinationParent = destinationFile.getParentFile(); - - if(!sourceFile.exists()) - { - throw (new FilesystemException("Cannot move file " + source + ", as it does not exist.")); - } - - ////////////////////////////////////////////////////////////////////////////////////// - // if the destination folder doesn't exist, try to make it - and fail if that fails // - ////////////////////////////////////////////////////////////////////////////////////// - if(!destinationParent.exists()) - { - LOG.debug("Making destination directory {} for move", destinationParent.getAbsolutePath()); - if(!destinationParent.mkdirs()) - { - throw (new FilesystemException("Failed to make destination directory " + destinationParent.getAbsolutePath() + " to move " + source + " into.")); - } - } - - if(!sourceFile.renameTo(destinationFile)) - { - throw (new FilesystemException("Failed to move (rename) file " + source + " to " + destination)); - } - } - } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index 91b5654b..74ddadc2 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -29,11 +29,14 @@ import java.io.InputStream; import java.util.Arrays; import java.util.Collections; import java.util.List; -import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; -import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -41,6 +44,9 @@ import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFile *******************************************************************************/ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction { + private static final Logger LOG = LogManager.getLogger(AbstractFilesystemAction.class); + + /******************************************************************************* ** List the files for this table. @@ -48,7 +54,8 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction @Override public List listFiles(QTableMetaData table, QBackendMetaData backendBase) { - String fullPath = getFullPath(table, backendBase); + // todo - needs rewritten to do globbing... + String fullPath = getFullBasePath(table, backendBase); File directory = new File(fullPath); File[] files = directory.listFiles(); @@ -74,15 +81,106 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction /******************************************************************************* - ** Add backend details to records about the file that they are in. + ** Write a file - to be implemented in module-specific subclasses. *******************************************************************************/ @Override - protected void addBackendDetailsToRecords(List recordsInFile, File file) + public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException { - recordsInFile.forEach(record -> + FileUtils.writeByteArrayToFile(new File(path), contents); + } + + + + /******************************************************************************* + ** Get a string that represents the full path to a file. + *******************************************************************************/ + @Override + protected String getFullPathForFile(File file) + { + return (file.getAbsolutePath()); + } + + + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + *******************************************************************************/ + @Override + public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + { + File file = new File(fileReference); + if(!file.exists()) { - record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, file.getAbsolutePath()); - }); + ////////////////////////////////////////////////////////////////////////////////////////////// + // if the file doesn't exist, just exit with noop. don't throw an error - that should only // + // happen if the "contract" of the method is broken, and the file still exists // + ////////////////////////////////////////////////////////////////////////////////////////////// + LOG.debug("Not deleting file [{}], because it does not exist.", file); + return; + } + + if(!file.delete()) + { + throw (new FilesystemException("Failed to delete file: " + fileReference)); + } + } + + + + /******************************************************************************* + ** Move a file from a source path, to a destination path. + ** + ** @param destination assumed to be a file path - not a directory + ** @throws FilesystemException if the delete is known to have failed + *******************************************************************************/ + @Override + public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException + { + File sourceFile = new File(source); + File destinationFile = new File(destination); + File destinationParent = destinationFile.getParentFile(); + + if(!sourceFile.exists()) + { + throw (new FilesystemException("Cannot move file " + source + ", as it does not exist.")); + } + + ////////////////////////////////////////////////////////////////////////////////////// + // if the destination folder doesn't exist, try to make it - and fail if that fails // + ////////////////////////////////////////////////////////////////////////////////////// + if(!destinationParent.exists()) + { + LOG.debug("Making destination directory {} for move", destinationParent.getAbsolutePath()); + if(!destinationParent.mkdirs()) + { + throw (new FilesystemException("Failed to make destination directory " + destinationParent.getAbsolutePath() + " to move " + source + " into.")); + } + } + + if(!sourceFile.renameTo(destinationFile)) + { + throw (new FilesystemException("Failed to move (rename) file " + source + " to " + destination)); + } + } + + + + /******************************************************************************* + ** e.g., with a base path of /foo/ + ** and a table path of /bar/ + ** and a file at /foo/bar/baz.txt + ** give us just the baz.txt part. + *******************************************************************************/ + @Override + public String stripBackendAndTableBasePathsFromFileName(File file, QBackendMetaData backend, QTableMetaData table) + { + String tablePath = getFullBasePath(table, backend); + String strippedPath = file.getAbsolutePath().replaceFirst(".*" + tablePath, ""); + return (strippedPath); } } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java index e35ad59b..4e096635 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java @@ -23,8 +23,6 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et import java.io.File; -import java.util.Set; -import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; @@ -43,7 +41,8 @@ import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; -import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -52,11 +51,20 @@ import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendD *******************************************************************************/ public class BasicETLCleanupSourceFilesFunction implements FunctionBody { + private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesFunction.class); + public static final String FIELD_MOVE_OR_DELETE = "moveOrDelete"; public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves"; + public static final String VALUE_MOVE = "move"; + public static final String VALUE_DELETE = "delete"; + public static final String FUNCTION_NAME = "cleanupSourceFiles"; + + /******************************************************************************* + ** Execute the function - using the request as input, and the result as output. + *******************************************************************************/ @Override public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException { @@ -70,17 +78,22 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function.")); } - Set sourceFiles = runFunctionRequest.getRecords().stream() - .map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH)) - .collect(Collectors.toSet()); + String sourceFilePaths = runFunctionRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS); + if(!StringUtils.hasContent(sourceFilePaths)) + { + LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS + "]"); + return; + } + + String[] sourceFiles = sourceFilePaths.split(","); for(String sourceFile : sourceFiles) { String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE); - if("delete".equals(moveOrDelete)) + if(VALUE_DELETE.equals(moveOrDelete)) { - filesystemModule.deleteFile(runFunctionRequest.getInstance(), table, sourceFile); + filesystemModule.getActionBase().deleteFile(runFunctionRequest.getInstance(), table, sourceFile); } - else if("move".equals(moveOrDelete)) + else if(VALUE_MOVE.equals(moveOrDelete)) { String destinationForMoves = runFunctionRequest.getValueString(FIELD_DESTINATION_FOR_MOVES); if(!StringUtils.hasContent(destinationForMoves)) @@ -89,11 +102,12 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody } File file = new File(sourceFile); String destinationPath = destinationForMoves + File.separator + file.getName(); - filesystemModule.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath); + filesystemModule.getActionBase().moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath); } else { - throw (new QException("Unexpected value [" + moveOrDelete + "] for field [" + FIELD_MOVE_OR_DELETE + "]. Must be either [move] or [delete].")); + throw (new QException("Unexpected value [" + moveOrDelete + "] for field [" + FIELD_MOVE_OR_DELETE + "]. " + + "Must be either [" + VALUE_MOVE + "] or [" + VALUE_DELETE + "].")); } } } @@ -101,12 +115,12 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody /******************************************************************************* - ** + ** define the metaData that describes this function *******************************************************************************/ public QFunctionMetaData defineFunctionMetaData() { return (new QFunctionMetaData() - .withName("cleanupSourceFiles") + .withName(FUNCTION_NAME) .withCode(new QCodeReference() .withName(this.getClass().getName()) .withCodeType(QCodeType.JAVA) diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java new file mode 100644 index 00000000..20de530f --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java @@ -0,0 +1,84 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic; + + +import java.util.Set; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; + + +/******************************************************************************* + ** Function body for collecting the file names that were discovered in the + ** Extract step. These will be lost during the transform, so we capture them here, + ** so that our Clean function can move or delete them. + ** + ** TODO - need unit test!! + *******************************************************************************/ +public class BasicETLCollectSourceFileNamesFunction implements FunctionBody +{ + public static final String FUNCTION_NAME = "collectSourceFileNames"; + public static final String FIELD_SOURCE_FILE_PATHS = "sourceFilePaths"; + + + + /******************************************************************************* + ** Execute the function - using the request as input, and the result as output. + *******************************************************************************/ + @Override + public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + { + Set sourceFiles = runFunctionRequest.getRecords().stream() + .map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH)) + .collect(Collectors.toSet()); + runFunctionResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles)); + } + + + + /******************************************************************************* + ** define the metaData that describes this function + *******************************************************************************/ + public QFunctionMetaData defineFunctionMetaData() + { + return (new QFunctionMetaData() + .withName(FUNCTION_NAME) + .withCode(new QCodeReference() + .withName(this.getClass().getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.FUNCTION)) + .withOutputMetaData(new QFunctionOutputMetaData() + .addField(new QFieldMetaData(FIELD_SOURCE_FILE_PATHS, QFieldType.STRING)))); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java new file mode 100644 index 00000000..be043bc7 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java @@ -0,0 +1,123 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync; + + +import java.io.File; +import java.io.InputStream; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Function body for collecting the file names that were discovered in the + ** Extract step. These will be lost during the transform, so we capture them here, + ** so that our Clean function can move or delete them. + ** + *******************************************************************************/ +public class FilesystemSyncFunction implements FunctionBody +{ + private static final Logger LOG = LogManager.getLogger(FilesystemSyncFunction.class); + + public static final String FUNCTION_NAME = "sync"; + + + + /******************************************************************************* + ** Execute the function - using the request as input, and the result as output. + *******************************************************************************/ + @Override + public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + { + QTableMetaData sourceTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE)); + QTableMetaData archiveTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE)); + QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE)); + + QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName()); + FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend); + Map sourceFiles = getFileNames(sourceModule.getActionBase(), sourceTable, sourceBackend); + + QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName()); + FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend); + Set archiveFiles = getFileNames(archiveModule.getActionBase(), archiveTable, archiveBackend).keySet(); + + QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName()); + FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend); + + for(Map.Entry sourceEntry : sourceFiles.entrySet()) + { + try + { + String sourceFileName = sourceEntry.getKey(); + if(!archiveFiles.contains(sourceFileName)) + { + LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]"); + InputStream inputStream = sourceModule.getActionBase().readFile(sourceEntry.getValue()); + byte[] bytes = inputStream.readAllBytes(); + + String archivePath = archiveModule.getActionBase().getFullBasePath(archiveTable, archiveBackend); + archiveModule.getActionBase().writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes); + + String processingPath = processingModule.getActionBase().getFullBasePath(processingTable, processingBackend); + processingModule.getActionBase().writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes); + } + } + catch(Exception e) + { + LOG.error("Error processing file: " + sourceEntry, e); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private Map getFileNames(AbstractBaseFilesystemAction actionBase, QTableMetaData table, QBackendMetaData backend) + { + List files = actionBase.listFiles(table, backend); + Map rs = new LinkedHashMap<>(); + + for(Object file : files) + { + String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(file, backend, table); + rs.put(fileName, file); + } + + return (rs); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java new file mode 100644 index 00000000..37313fb2 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java @@ -0,0 +1,78 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync; + + +import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; + + +/******************************************************************************* + ** Definition for Filesystem sync process. + ** + ** Job is to: + ** - list all files in the source table. + ** - list all files in the archive table. + ** - if any files exist in the source, but not in the archive, then: + ** - copy the file to both the archive and the processing table. + ** + ** The idea being, that the source is read-only, and we want to move files out of + ** processing after they've been processed - and the archive is what we can have + ** in-between the two. + *******************************************************************************/ +public class FilesystemSyncProcess +{ + public static final String PROCESS_NAME = "filesystem.sync"; + + public static final String FIELD_SOURCE_TABLE = "sourceTable"; + public static final String FIELD_ARCHIVE_TABLE = "archiveTable"; + public static final String FIELD_PROCESSING_TABLE = "processingTable"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QProcessMetaData defineProcessMetaData() + { + QFunctionMetaData syncFunction = new QFunctionMetaData() + .withName(FilesystemSyncFunction.FUNCTION_NAME) + .withCode(new QCodeReference() + .withName(FilesystemSyncFunction.class.getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.FUNCTION)) + .withInputData(new QFunctionInputMetaData() + .addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)) + .addField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING)) + .addField(new QFieldMetaData(FIELD_PROCESSING_TABLE, QFieldType.STRING))); + + return new QProcessMetaData() + .withName(PROCESS_NAME) + .addFunction(syncFunction); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java index 33bffb0b..38a54535 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java @@ -22,24 +22,23 @@ package com.kingsrook.qqq.backend.module.filesystem.s3; +import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; -import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; -import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; /******************************************************************************* @@ -47,7 +46,17 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; *******************************************************************************/ public class S3BackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface { - private S3Utils s3Utils; + + + /******************************************************************************* + ** For filesystem backends, get the module-specific action base-class, that helps + ** with functions like listing and deleting files. + *******************************************************************************/ + @Override + public AbstractBaseFilesystemAction getActionBase() + { + return (new AbstractS3Action()); + } @@ -84,43 +93,6 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke - /******************************************************************************* - ** In contrast with the DeleteAction, which deletes RECORDS - this is a - ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE - ** e.g., for post-ETL. - ** - ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit - *******************************************************************************/ - @Override - public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException - { - QBackendMetaData backend = instance.getBackend(table.getBackendName()); - String bucketName = ((S3BackendMetaData) backend).getBucketName(); - - getS3Utils().deleteObject(bucketName, fileReference); - } - - - - - /******************************************************************************* - ** In contrast with the DeleteAction, which deletes RECORDS - this is a - ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE - ** e.g., for post-ETL. - ** - ** @throws FilesystemException if the move is known to have failed - *******************************************************************************/ - @Override - public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException - { - QBackendMetaData backend = instance.getBackend(table.getBackendName()); - String bucketName = ((S3BackendMetaData) backend).getBucketName(); - - getS3Utils().moveObject(bucketName, source, destination); - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -163,29 +135,4 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke return (new S3DeleteAction()); } - - - /******************************************************************************* - ** Setter for s3Utils - *******************************************************************************/ - public void setS3Utils(S3Utils s3Utils) - { - this.s3Utils = s3Utils; - } - - - - /******************************************************************************* - ** Internal accessor for the s3Utils object - should always use this, not the field. - *******************************************************************************/ - private S3Utils getS3Utils() - { - if(s3Utils == null) - { - s3Utils = new S3Utils(); - } - - return s3Utils; - } - } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 1fb11d34..281d3583 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -25,14 +25,22 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; import java.io.IOException; import java.io.InputStream; import java.util.List; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.actions.AbstractQTableRequest; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; -import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -40,10 +48,46 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; *******************************************************************************/ public class AbstractS3Action extends AbstractBaseFilesystemAction { + private static final Logger LOG = LogManager.getLogger(AbstractS3Action.class); + private S3Utils s3Utils; + /******************************************************************************* + ** Setup the s3 utils object to be used for this action. + *******************************************************************************/ + @Override + protected void preAction(AbstractQTableRequest tableRequest) + { + super.preAction(tableRequest); + + if(s3Utils != null) + { + LOG.debug("s3Utils object is already set - not re-setting it."); + return; + } + + S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, tableRequest.getBackend()); + AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard(); + + if(StringUtils.hasContent(s3BackendMetaData.getAccessKey()) && StringUtils.hasContent(s3BackendMetaData.getSecretKey())) + { + BasicAWSCredentials credentials = new BasicAWSCredentials(s3BackendMetaData.getAccessKey(), s3BackendMetaData.getSecretKey()); + clientBuilder.setCredentials(new AWSStaticCredentialsProvider(credentials)); + } + + if(StringUtils.hasContent(s3BackendMetaData.getRegion())) + { + clientBuilder.setRegion(s3BackendMetaData.getRegion()); + } + + s3Utils = new S3Utils(); + s3Utils.setAmazonS3(clientBuilder.build()); + } + + + /******************************************************************************* ** Set the S3Utils object. *******************************************************************************/ @@ -75,17 +119,17 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction listFiles(QTableMetaData table, QBackendMetaData backendBase) { - S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase); + S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase); + AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); - String fullPath = getFullPath(table, backendBase); + String fullPath = getFullBasePath(table, backendBase); String bucketName = s3BackendMetaData.getBucketName(); + String glob = tableDetails.getGlob(); //////////////////////////////////////////////////////////////////// - // todo - read metadata to decide if we should include subfolders // // todo - look at metadata to configure the s3 client here? // //////////////////////////////////////////////////////////////////// - boolean includeSubfolders = false; - return getS3Utils().listObjectsInBucketAtPath(bucketName, fullPath, includeSubfolders); + return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob); } @@ -102,15 +146,77 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction recordsInFile, S3ObjectSummary s3ObjectSummary) + public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException { - recordsInFile.forEach(record -> - { - record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, s3ObjectSummary.getKey()); - }); + path = stripDuplicatedSlashes(path); + String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName(); + getS3Utils().writeFile(bucketName, path, contents); + } + + + /******************************************************************************* + ** Get a string that represents the full path to a file. + *******************************************************************************/ + @Override + protected String getFullPathForFile(S3ObjectSummary s3ObjectSummary) + { + return (s3ObjectSummary.getKey()); + } + + + + /******************************************************************************* + ** e.g., with a base path of /foo/ + ** and a table path of /bar/ + ** and a file at /foo/bar/baz.txt + ** give us just the baz.txt part. + *******************************************************************************/ + @Override + public String stripBackendAndTableBasePathsFromFileName(S3ObjectSummary file, QBackendMetaData backend, QTableMetaData table) + { + String tablePath = getFullBasePath(table, backend); + String strippedPath = file.getKey().replaceFirst(".*" + tablePath, ""); + return (strippedPath); + } + + + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + *******************************************************************************/ + @Override + public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + { + QBackendMetaData backend = instance.getBackend(table.getBackendName()); + String bucketName = ((S3BackendMetaData) backend).getBucketName(); + + getS3Utils().deleteObject(bucketName, fileReference); + } + + + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @param destination assumed to be a file path - not a directory + ** @throws FilesystemException if the move is known to have failed + *******************************************************************************/ + @Override + public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException + { + QBackendMetaData backend = instance.getBackend(table.getBackendName()); + String bucketName = ((S3BackendMetaData) backend).getBucketName(); + + getS3Utils().moveObject(bucketName, source, destination); } } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java index 70bbe0ff..5f064b39 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java @@ -22,16 +22,20 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata; +import com.kingsrook.qqq.backend.core.model.metadata.QSecretReader; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; /******************************************************************************* - ** (local) Filesystem backend meta data. + ** S3 backend meta data. *******************************************************************************/ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData { private String bucketName; + private String accessKey; + private String secretKey; + private String region; @@ -79,4 +83,124 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData return (T) this; } + + + /******************************************************************************* + ** Getter for accessKey + ** + *******************************************************************************/ + public String getAccessKey() + { + return accessKey; + } + + + + /******************************************************************************* + ** Setter for accessKey + ** + *******************************************************************************/ + public void setAccessKey(String accessKey) + { + this.accessKey = accessKey; + } + + + + /******************************************************************************* + ** Fluent setter for accessKey + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withAccessKey(String accessKey) + { + this.accessKey = accessKey; + return (T) this; + } + + + + /******************************************************************************* + ** Getter for secretKey + ** + *******************************************************************************/ + public String getSecretKey() + { + return secretKey; + } + + + + /******************************************************************************* + ** Setter for secretKey + ** + *******************************************************************************/ + public void setSecretKey(String secretKey) + { + this.secretKey = secretKey; + } + + + + /******************************************************************************* + ** Fluent setter for secretKey + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withSecretKey(String secretKey) + { + this.secretKey = secretKey; + return (T) this; + } + + + + /******************************************************************************* + ** Getter for region + ** + *******************************************************************************/ + public String getRegion() + { + return region; + } + + + + /******************************************************************************* + ** Setter for region + ** + *******************************************************************************/ + public void setRegion(String region) + { + this.region = region; + } + + + + /******************************************************************************* + ** Fluent setter for region + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withRegion(String region) + { + this.region = region; + return (T) this; + } + + + + /******************************************************************************* + ** Called by the QInstanceEnricher - to do backend-type-specific enrichments. + ** Original use case is: reading secrets into fields (e.g., passwords). + *******************************************************************************/ + @Override + public void enrich() + { + super.enrich(); + QSecretReader secretReader = new QSecretReader(); + accessKey = secretReader.readSecret(accessKey); + secretKey = secretReader.readSecret(secretKey); + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java index 65a25367..adc2b374 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java @@ -22,7 +22,12 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.utils; +import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.net.URI; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.PathMatcher; import java.util.ArrayList; import java.util.List; import com.amazonaws.regions.Regions; @@ -30,8 +35,10 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.ListObjectsV2Request; import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -47,25 +54,57 @@ public class S3Utils { private static final Logger LOG = LogManager.getLogger(S3Utils.class); - private AmazonS3 s3; + private AmazonS3 amazonS3; /******************************************************************************* - ** List the objects in an S3 bucket at a given path + ** List the objects in an S3 bucket matching a glob, per: + ** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String) *******************************************************************************/ - public List listObjectsInBucketAtPath(String bucketName, String fullPath, boolean includeSubfolders) + public List listObjectsInBucketMatchingGlob(String bucketName, String path, String glob) { ////////////////////////////////////////////////////////////////////////////////////////////////// // s3 list requests find nothing if the path starts with a /, so strip away any leading slashes // // also strip away trailing /'s, for consistent known paths. // // also normalize any duplicated /'s to a single /. // ////////////////////////////////////////////////////////////////////////////////////////////////// - fullPath = fullPath.replaceFirst("^/+", "").replaceFirst("/+$", "").replaceAll("//+", "/"); + path = path.replaceFirst("^/+", "").replaceFirst("/+$", "").replaceAll("//+", "/"); + String prefix = path; + + // todo - maybe this is some error - that the user put a * in the path instead of the glob? + if(prefix.indexOf('*') > -1) + { + prefix = prefix.substring(0, prefix.indexOf('*')); + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob. // + // That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. // + // Anyway, add a trailing /* to globs with no wildcards (or just a '*' if it's a request for the root ("")) // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(glob == null) + { + glob = ""; + } + if(!glob.contains("*")) + { + if(glob.equals("")) + { + glob += "*"; + } + else + { + glob += "/*"; + } + } + + String pathMatcherArg = AbstractFilesystemAction.stripDuplicatedSlashes("glob:/" + path + "/" + glob); + PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher(pathMatcherArg); ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request() .withBucketName(bucketName) - .withPrefix(fullPath); + .withPrefix(prefix); ListObjectsV2Result listObjectsV2Result = null; List rs = new ArrayList<>(); @@ -76,7 +115,8 @@ public class S3Utils { listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken()); } - listObjectsV2Result = getS3().listObjectsV2(listObjectsV2Request); + LOG.info("Listing bucket=" + bucketName + ", path=" + path); + listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request); ////////////////////////////////// // put files in the result list // @@ -85,26 +125,29 @@ public class S3Utils { String key = objectSummary.getKey(); - ////////////////// - // skip folders // - ////////////////// + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // it looks like keys in s3 can have duplicated /'s - so normalize those, to create a "sane" result // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + key = key.replaceAll("//+", "/"); + + //////////////////////////////////////////////////////////////////////////////// + // always skip folders // + // this seemed to fire when it was first written, but not in our unit tests - // + // is this a difference with real s3 vs. localstack possibly? // + //////////////////////////////////////////////////////////////////////////////// if(key.endsWith("/")) { LOG.debug("Skipping file [{}] because it is a folder", key); continue; } - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // if we're not supposed to include subfolders, check the path on this file, and only include it if it matches the request // - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(!includeSubfolders) + /////////////////////////////////////////// + // skip files that do not match the glob // + /////////////////////////////////////////// + if(!pathMatcher.matches(Path.of(URI.create("file:///" + key)))) { - String prefix = key.substring(0, key.lastIndexOf("/")); - if(!prefix.equals(fullPath)) - { - LOG.debug("Skipping file [{}] in a sub-folder [{}] != [{}]", key, prefix, fullPath); - continue; - } + LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob); + continue; } rs.add(objectSummary); @@ -122,7 +165,17 @@ public class S3Utils *******************************************************************************/ public InputStream getObjectAsInputStream(S3ObjectSummary s3ObjectSummary) { - return getS3().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()).getObjectContent(); + return getAmazonS3().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()).getObjectContent(); + } + + + + /******************************************************************************* + ** Write a file + *******************************************************************************/ + public void writeFile(String bucket, String key, byte[] contents) + { + getAmazonS3().putObject(bucket, key, new ByteArrayInputStream(contents), new ObjectMetadata()); } @@ -137,7 +190,7 @@ public class S3Utils ////////////////////////////////////////////////////////////////////////////////////////////// try { - getS3().deleteObject(bucketName, key); + getAmazonS3().deleteObject(bucketName, key); } catch(Exception e) { @@ -157,8 +210,8 @@ public class S3Utils ////////////////////////////////////////////////////////////////////////////////////////////// try { - getS3().copyObject(bucketName, source, bucketName, destination); - getS3().deleteObject(bucketName, source); + getAmazonS3().copyObject(bucketName, source, bucketName, destination); + getAmazonS3().deleteObject(bucketName, source); } catch(Exception e) { @@ -171,9 +224,9 @@ public class S3Utils /******************************************************************************* ** Setter for AmazonS3 client object. *******************************************************************************/ - public void setAmazonS3(AmazonS3 s3) + public void setAmazonS3(AmazonS3 amazonS3) { - this.s3 = s3; + this.amazonS3 = amazonS3; } @@ -181,14 +234,14 @@ public class S3Utils /******************************************************************************* ** Getter for AmazonS3 client object. *******************************************************************************/ - public AmazonS3 getS3() + public AmazonS3 getAmazonS3() { - if(s3 == null) + if(amazonS3 == null) { - s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build(); + // TODO - get this (and other props?) from backend meta data + amazonS3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build(); } - return s3; + return amazonS3; } - } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index d94da070..676e3f96 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -33,6 +33,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.mock.MockAuthenticationModule; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; @@ -56,6 +58,10 @@ public class TestUtils /////////////////////////////////////////////////////////////////// public static final String BASE_PATH = "/tmp/filesystem-tests"; + ////////////////////////////////////////////////////////////////////////////// + // Used to make each test method have a unique folder path, more or less... // + // See methods that work with it. // + ////////////////////////////////////////////////////////////////////////////// private static int testInstanceCounter = 0; @@ -110,7 +116,7 @@ public class TestUtils QInstance qInstance = new QInstance(); qInstance.setAuthentication(defineAuthentication()); qInstance.addBackend(defineLocalFilesystemBackend()); - qInstance.addTable(defineLocalFilesystemCSVPersonTable()); + qInstance.addTable(defineLocalFilesystemJSONPersonTable()); qInstance.addBackend(defineS3Backend()); qInstance.addTable(defineS3CSVPersonTable()); @@ -149,7 +155,7 @@ public class TestUtils /******************************************************************************* ** *******************************************************************************/ - public static QTableMetaData defineLocalFilesystemCSVPersonTable() + public static QTableMetaData defineLocalFilesystemJSONPersonTable() { return new QTableMetaData() .withName(TABLE_NAME_PERSON_LOCAL_FS) @@ -164,9 +170,9 @@ public class TestUtils .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) .withField(new QFieldMetaData("email", QFieldType.STRING)) .withBackendDetails(new FilesystemTableBackendDetails() - .withPath("persons") - .withRecordFormat("json") - .withCardinality("many") + .withBasePath("persons") + .withRecordFormat(RecordFormat.JSON) + .withCardinality(Cardinality.MANY) ); } @@ -203,8 +209,8 @@ public class TestUtils .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) .withField(new QFieldMetaData("email", QFieldType.STRING)) .withBackendDetails(new S3TableBackendDetails() - .withRecordFormat("csv") - .withCardinality("many") + .withRecordFormat(RecordFormat.CSV) + .withCardinality(Cardinality.MANY) ); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java index d0d43329..9d172573 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java @@ -78,7 +78,7 @@ public class FilesystemBackendModuleTest List filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); - filesystemBackendModule.deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath()); + filesystemBackendModule.getActionBase().deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath()); List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); Assertions.assertEquals(filesBeforeDelete.size() - 1, filesAfterDelete.size(), @@ -98,12 +98,13 @@ public class FilesystemBackendModuleTest ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count // - // note, we'd like to detect the non-delete, but there's no such info back from aws it appears? // + // note, our contract is that as long as the file doesn't exist after calling delete (e.g., if it wasn't there // + // to begin with, then we're okay, and don't expect an exception // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// List filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); - filesystemBackendModule.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + filesystemBackendModule.getActionBase().deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); Assertions.assertEquals(filesBeforeDelete.size(), filesAfterDelete.size(), @@ -133,7 +134,7 @@ public class FilesystemBackendModuleTest String originalFilePath = filesBeforeMove.get(0).getAbsolutePath(); String movedFilePath = originalFilePath.replace(basePath, subPath); - filesystemBackendModule.moveFile(qInstance, table, originalFilePath, movedFilePath); + filesystemBackendModule.getActionBase().moveFile(qInstance, table, originalFilePath, movedFilePath); List filesAfterMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); Assertions.assertEquals(filesBeforeMove.size() - 1, filesAfterMove.size(), "Should be one fewer file in the listing after moving one."); @@ -141,7 +142,7 @@ public class FilesystemBackendModuleTest ////////////////////////////////////////////////////////////////////////// // move the file back and assert that the count goes back to the before // ////////////////////////////////////////////////////////////////////////// - filesystemBackendModule.moveFile(qInstance, table, movedFilePath, originalFilePath); + filesystemBackendModule.getActionBase().moveFile(qInstance, table, movedFilePath, originalFilePath); List filesAfterMoveBack = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); Assertions.assertEquals(filesBeforeMove.size(), filesAfterMoveBack.size(), "Should be original number of files after moving back"); @@ -166,7 +167,7 @@ public class FilesystemBackendModuleTest Assertions.assertThrows(FilesystemException.class, () -> { FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); - filesystemBackendModule.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, movedFilePath); + filesystemBackendModule.getActionBase().moveFile(qInstance, table, PATH_THAT_WONT_EXIST, movedFilePath); }); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java index 9e9b38a2..755ab477 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java @@ -29,6 +29,8 @@ import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import static org.junit.jupiter.api.Assertions.fail; @@ -40,6 +42,28 @@ import static org.junit.jupiter.api.Assertions.fail; public class FilesystemActionTest { + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + public void beforeEach() throws Exception + { + primeFilesystem(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + public void afterEach() throws Exception + { + cleanFilesystem(); + } + + + /******************************************************************************* ** Set up the file system *******************************************************************************/ @@ -61,14 +85,17 @@ public class FilesystemActionTest + /******************************************************************************* + ** Write some data files into the directory for the filesystem module. + *******************************************************************************/ private void writePersonFiles(File baseDirectory) throws IOException { String fullPath = baseDirectory.getAbsolutePath(); - if (TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details) + if (TestUtils.defineLocalFilesystemJSONPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details) { - if (StringUtils.hasContent(details.getPath())) + if (StringUtils.hasContent(details.getBasePath())) { - fullPath += File.separatorChar + details.getPath(); + fullPath += File.separatorChar + details.getBasePath(); } } fullPath += File.separatorChar; diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java index f624cb5a..de3e21d8 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java @@ -28,9 +28,7 @@ import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -40,28 +38,6 @@ import org.junit.jupiter.api.Test; public class FilesystemQueryActionTest extends FilesystemActionTest { - /******************************************************************************* - ** - *******************************************************************************/ - @BeforeEach - public void beforeEach() throws Exception - { - super.primeFilesystem(); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @AfterEach - public void afterEach() throws Exception - { - super.cleanFilesystem(); - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -85,7 +61,7 @@ public class FilesystemQueryActionTest extends FilesystemActionTest { QueryRequest queryRequest = new QueryRequest(); queryRequest.setInstance(TestUtils.defineInstance()); - queryRequest.setTableName(TestUtils.defineLocalFilesystemCSVPersonTable().getName()); + queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName()); return queryRequest; } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java index e8b24695..7c6515f6 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java @@ -71,7 +71,7 @@ class FilesystemBackendMetaDataTest QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json); assertThat(deserialized).usingRecursiveComparison() - .ignoringFields("hasBeenValidated") + .ignoringFields("hasBeenValidated") // note, this field is @JsonIgnore .isEqualTo(qInstance); } } \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java index 183bd1d1..8da9dbda 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java @@ -23,20 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et import java.io.File; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.UUID; import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; -import com.kingsrook.qqq.backend.core.callbacks.NoopCallback; import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; -import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; -import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import org.apache.commons.io.FileUtils; @@ -56,17 +55,11 @@ public class BasicETLCleanupSourceFilesFunctionTest ** *******************************************************************************/ @Test - public void testDelete() throws Exception + public void testDelete1Record1File() throws Exception { QInstance qInstance = TestUtils.defineInstance(); String filePath = getRandomFilePathPersonTable(qInstance); - - RunFunctionResult runFunctionResult = runFunction(qInstance, filePath, Map.of( - BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, "delete", - BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, "/tmp/trash")); - - assertNull(runFunctionResult.getError()); - assertFalse(new File(filePath).exists(), "File should have been deleted."); + testDelete(qInstance, List.of(filePath)); } @@ -75,21 +68,11 @@ public class BasicETLCleanupSourceFilesFunctionTest ** *******************************************************************************/ @Test - public void testMove() throws Exception + public void testDelete2Records1File() throws Exception { QInstance qInstance = TestUtils.defineInstance(); String filePath = getRandomFilePathPersonTable(qInstance); - - String trashDir = File.separator + "tmp" + File.separator + "trash"; - RunFunctionResult runFunctionResult = runFunction(qInstance, filePath, Map.of( - BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, "move", - BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir)); - - assertNull(runFunctionResult.getError()); - assertFalse(new File(filePath).exists(), "File should have been moved."); - - String movedPath = trashDir + File.separator + (new File(filePath).getName()); - assertTrue(new File(movedPath).exists(), "File should have been moved."); + testDelete(qInstance, List.of(filePath, filePath)); } @@ -97,25 +80,127 @@ public class BasicETLCleanupSourceFilesFunctionTest /******************************************************************************* ** *******************************************************************************/ - private RunFunctionResult runFunction(QInstance qInstance, String filePath, Map values) throws Exception + @Test + public void testDelete2Record2File() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath1 = getRandomFilePathPersonTable(qInstance); + String filePath2 = getRandomFilePathPersonTable(qInstance); + testDelete(qInstance, List.of(filePath1, filePath2)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMove1Record1File() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath = getRandomFilePathPersonTable(qInstance); + testMove(qInstance, List.of(filePath)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMove2Records1File() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath = getRandomFilePathPersonTable(qInstance); + testMove(qInstance, List.of(filePath, filePath)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMove2Record2File() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath1 = getRandomFilePathPersonTable(qInstance); + String filePath2 = getRandomFilePathPersonTable(qInstance); + testMove(qInstance, List.of(filePath1, filePath2)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void testDelete(QInstance qInstance, List filePaths) throws Exception + { + RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of( + BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_DELETE, + // todo - even though this field isn't needed, since we gave a value of "delete" + // the RunFunctionAction considers any missing input to be an error... + BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, "")); + + assertNull(runFunctionResult.getError()); + for(String filePath : filePaths) + { + assertFalse(new File(filePath).exists(), "File should have been deleted."); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void testMove(QInstance qInstance, List filePaths) throws Exception + { + String trashDir = File.separator + "tmp" + File.separator + "trash"; + RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of( + BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_MOVE, + BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir)); + + assertNull(runFunctionResult.getError()); + + for(String filePath : filePaths) + { + assertFalse(new File(filePath).exists(), "File should have been moved."); + + String movedPath = trashDir + File.separator + (new File(filePath).getName()); + assertTrue(new File(movedPath).exists(), "File should have been moved."); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private RunFunctionResult runFunction(QInstance qInstance, List filePaths, Map values) throws Exception { QFunctionMetaData qFunctionMetaData = new BasicETLCleanupSourceFilesFunction().defineFunctionMetaData(); QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addFunction(qFunctionMetaData); qInstance.addProcess(qProcessMetaData); - File file = new File(filePath); - FileUtils.writeStringToFile(file, "content"); + HashSet filePathsSet = new HashSet<>(filePaths); + for(String filePath : filePathsSet) + { + File file = new File(filePath); + FileUtils.writeStringToFile(file, "content"); + } - List records = List.of(new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)); + // List records = filePaths.stream() + // .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList(); RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); runFunctionRequest.setFunctionName(qFunctionMetaData.getName()); runFunctionRequest.setProcessName(qProcessMetaData.getName()); - runFunctionRequest.setCallback(new NoopCallback()); - runFunctionRequest.setRecords(records); + // runFunctionRequest.setRecords(records); runFunctionRequest.setSession(TestUtils.getMockSession()); runFunctionRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS); runFunctionRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3); + runFunctionRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet)); for(Map.Entry entry : values.entrySet()) { @@ -135,7 +220,7 @@ public class BasicETLCleanupSourceFilesFunctionTest { FilesystemBackendMetaData backend = (FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS); FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS).getBackendDetails(); - String tablePath = backend.getBasePath() + File.separator + backendDetails.getPath(); + String tablePath = backend.getBasePath() + File.separator + backendDetails.getBasePath(); String filePath = tablePath + File.separator + UUID.randomUUID(); return filePath; } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java new file mode 100644 index 00000000..2064617c --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java @@ -0,0 +1,266 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync; + + +import java.util.List; +import java.util.stream.Collectors; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; +import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; +import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModuleSubclassForTest; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for FilesystemSyncProcess using S3 backend + *******************************************************************************/ +class FilesystemSyncProcessS3Test extends BaseS3Test +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws Exception + { + QBackendModuleDispatcher.registerBackendModule(new S3BackendModuleSubclassForTest()); + QInstance qInstance = TestUtils.defineInstance(); + + String sourceBucket = "source-bucket"; + String archiveBucket = "archive-bucket"; + String processingBucket = "processing-bucket"; + + getAmazonS3().createBucket(sourceBucket); + getAmazonS3().createBucket(archiveBucket); + getAmazonS3().createBucket(processingBucket); + + S3BackendMetaData sourceBackend = defineBackend(qInstance, "source", sourceBucket); + S3BackendMetaData archiveBackend = defineBackend(qInstance, "archive", archiveBucket); + S3BackendMetaData processingBackend = defineBackend(qInstance, "processing", processingBucket); + + QTableMetaData sourceTable = defineTable(qInstance, "source", sourceBackend, "source", "*/l3/*.csv"); + QTableMetaData archiveTable = defineTable(qInstance, "archive", archiveBackend, "archive", "*/l3/*.csv"); + QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv"); + + QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); + QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); + qInstance.addProcess(process); + + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); + + /////////////////////////// + // write some test files // + /////////////////////////// + writeTestFile(sourceBackend, sourceTable, "foo/l3/1.csv", "x"); + writeTestFile(sourceBackend, sourceTable, "bar/l3/2.csv", "x"); + writeTestFile(archiveBackend, archiveTable, "foo/l3/1.csv", "x"); + + printTableListing(sourceBackend, sourceTable); + printTableListing(archiveBackend, archiveTable); + printTableListing(processingBackend, processingTable); + + ////////////////////// + // run the function // + ////////////////////// + RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); + runFunctionRequest.setFunctionName(function.getName()); + runFunctionRequest.setProcessName(process.getName()); + runFunctionRequest.setSession(TestUtils.getMockSession()); + + RunFunctionAction runFunctionAction = new RunFunctionAction(); + RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); + System.out.println(runFunctionResult); + + printTableListing(sourceBackend, sourceTable); + printTableListing(archiveBackend, archiveTable); + printTableListing(processingBackend, processingTable); + + assertTableListing(archiveBackend, archiveTable, "root/archive/foo/l3/1.csv", "root/archive/bar/l3/2.csv"); + assertTableListing(processingBackend, processingTable, "root/processing/bar/l3/2.csv"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testArchiveAndProcessingInSameBucket() throws Exception + { + QBackendModuleDispatcher.registerBackendModule(new S3BackendModuleSubclassForTest()); + QInstance qInstance = TestUtils.defineInstance(); + + String vendorBucket = "vendor-bucket"; + String localBucket = "local-bucket"; + + getAmazonS3().createBucket(vendorBucket); + getAmazonS3().createBucket(localBucket); + + S3BackendMetaData vendorBackend = defineBackend(qInstance, "source", vendorBucket); + S3BackendMetaData localBackend = defineBackend(qInstance, "archive", localBucket); + + QTableMetaData sourceTable = defineTable(qInstance, "source", vendorBackend, "source", "*/l3/*.csv"); + QTableMetaData archiveTable = defineTable(qInstance, "archive", localBackend, "archive", "*/l3/*.csv"); + QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv"); + + QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); + QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); + qInstance.addProcess(process); + + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); + + /////////////////////////// + // write some test files // + /////////////////////////// + writeTestFile(vendorBackend, sourceTable, "foo/l3/1.csv", "x"); + writeTestFile(vendorBackend, sourceTable, "bar/l3/2.csv", "x"); + writeTestFile(localBackend, archiveTable, "foo/l3/1.csv", "x"); + + printTableListing(vendorBackend, sourceTable); + printTableListing(localBackend, archiveTable); + printTableListing(localBackend, processingTable); + + ////////////////////// + // run the function // + ////////////////////// + RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); + runFunctionRequest.setFunctionName(function.getName()); + runFunctionRequest.setProcessName(process.getName()); + runFunctionRequest.setSession(TestUtils.getMockSession()); + + RunFunctionAction runFunctionAction = new RunFunctionAction(); + RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); + System.out.println(runFunctionResult); + + printTableListing(vendorBackend, sourceTable); + printTableListing(localBackend, archiveTable); + printTableListing(localBackend, processingTable); + + assertTableListing(localBackend, archiveTable, "root/archive/foo/l3/1.csv", "root/archive/bar/l3/2.csv"); + assertTableListing(localBackend, processingTable, "root/processing/bar/l3/2.csv"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException + { + S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend); + AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase(); + + List s3ObjectSummaries = actionBase.listFiles(table, backend); + assertEquals(paths.length, s3ObjectSummaries.size(), "Expected number of files in table: " + table.getName()); + for(String path : paths) + { + assertTrue(s3ObjectSummaries.stream().anyMatch(s3o -> s3o.getKey().equals(path)), + "Path [" + path + "] should be in the listing, but was not. Full listing is: " + + s3ObjectSummaries.stream().map(S3ObjectSummary::getKey).collect(Collectors.joining(","))); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException + { + S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend); + AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase(); + + System.out.println("Files in: " + table.getName()); + actionBase.listFiles(table, backend).forEach(o -> System.out.println(o.getKey())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeTestFile(S3BackendMetaData backend, QTableMetaData table, String name, String content) throws Exception + { + S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend); + AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase(); + String fullPath = actionBase.getFullBasePath(table, backend); + + actionBase.writeFile(backend, fullPath + "/" + name, content.getBytes()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private S3BackendMetaData defineBackend(QInstance qInstance, String which, String bucketName) + { + QBackendMetaData backendMetaData = new S3BackendMetaData() + .withBucketName(bucketName) + .withBasePath("root") + .withName("backend-" + which); + qInstance.addBackend(backendMetaData); + return (S3BackendMetaData) backendMetaData; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineTable(QInstance qInstance, String which, QBackendMetaData backend, String path, String glob) + { + QTableMetaData qTableMetaData = new QTableMetaData() + .withName("table-" + which) + .withBackendName(backend.getName()) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withBackendDetails(new S3TableBackendDetails() + .withBasePath(path) + .withGlob(glob)); + qInstance.addTable(qTableMetaData); + return (qTableMetaData); + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java new file mode 100644 index 00000000..73254757 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java @@ -0,0 +1,119 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync; + + +import java.io.File; +import java.io.IOException; +import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for FilesystemSyncProcess + *******************************************************************************/ +class FilesystemSyncProcessTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws Exception + { + QTableMetaData sourceTable = defineTable("source"); + QTableMetaData archiveTable = defineTable("archive"); + QTableMetaData processingTable = defineTable("processing"); + QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); + QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); + + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); + + QInstance qInstance = TestUtils.defineInstance(); + qInstance.addTable(sourceTable); + qInstance.addTable(archiveTable); + qInstance.addTable(processingTable); + qInstance.addProcess(process); + + /////////////////////////// + // write some test files // + /////////////////////////// + String basePath = ((FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS)).getBasePath(); + writeTestFile(basePath, sourceTable, "1.txt", "x"); + writeTestFile(basePath, sourceTable, "2.txt", "x"); + writeTestFile(basePath, archiveTable, "2.txt", "x"); + + ////////////////////// + // run the function // + ////////////////////// + RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); + runFunctionRequest.setFunctionName(function.getName()); + runFunctionRequest.setProcessName(process.getName()); + runFunctionRequest.setSession(TestUtils.getMockSession()); + + RunFunctionAction runFunctionAction = new RunFunctionAction(); + RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); + System.out.println(runFunctionResult); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeTestFile(String basePath, QTableMetaData table, String name, String content) throws IOException + { + String path = ((FilesystemTableBackendDetails) table.getBackendDetails()).getBasePath(); + File file = new File(basePath + "/" + path + "/" + name); + FileUtils.writeStringToFile(file, content); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineTable(String subPath) + { + return new QTableMetaData() + .withName("table-" + subPath) + .withBackendName(TestUtils.BACKEND_NAME_LOCAL_FS) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withBackendDetails(new FilesystemTableBackendDetails() + .withBasePath(subPath)); + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java index 3563d91c..490a10b7 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java @@ -59,6 +59,7 @@ public class BaseS3Test amazonS3.putObject(BUCKET_NAME, "0.csv", getCSVHeader()); amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/1.csv", getCSVData1()); amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2()); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test"); amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3()); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleSubclassForTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleSubclassForTest.java new file mode 100644 index 00000000..84e5189f --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleSubclassForTest.java @@ -0,0 +1,53 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3; + + +import com.amazonaws.services.s3.model.S3ObjectSummary; +import cloud.localstack.awssdkv1.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; +import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; + + +/******************************************************************************* + ** Subclass of the S3Backend module, meant for use in unit tests, if/where we + ** need to make sure we use the localstack version of the S3 client. + *******************************************************************************/ +public class S3BackendModuleSubclassForTest extends S3BackendModule +{ + + /******************************************************************************* + ** Seed the AbstractS3Action with an s3Utils object that has the localstack + ** s3 client in it + *******************************************************************************/ + @Override + public AbstractBaseFilesystemAction getActionBase() + { + AbstractS3Action actionBase = (AbstractS3Action) super.getActionBase(); + S3Utils s3Utils = new S3Utils(); + s3Utils.setAmazonS3(TestUtils.getClientS3()); + actionBase.setS3Utils(s3Utils); + return (actionBase); + } + +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java index e0cb0489..44d1796d 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -54,13 +55,14 @@ public class S3BackendModuleTest extends BaseS3Test ///////////////////////////////////////////////////////////////////////////////////////////// // first list the files - then delete one, then re-list, and assert that we have one fewer // ///////////////////////////////////////////////////////////////////////////////////////////// - List s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + List s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); - S3BackendModule s3BackendModule = new S3BackendModule(); - s3BackendModule.setS3Utils(getS3Utils()); - s3BackendModule.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey()); + S3BackendModule s3BackendModule = new S3BackendModule(); + AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); + actionBase.setS3Utils(getS3Utils()); + actionBase.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey()); - List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(), "Should be one fewer file listed after deleting one."); } @@ -80,13 +82,14 @@ public class S3BackendModuleTest extends BaseS3Test // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count // // note, we'd like to detect the non-delete, but there's no such info back from aws it appears? // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// - List s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + List s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); - S3BackendModule s3BackendModule = new S3BackendModule(); - s3BackendModule.setS3Utils(getS3Utils()); - s3BackendModule.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + S3BackendModule s3BackendModule = new S3BackendModule(); + AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); + actionBase.setS3Utils(getS3Utils()); + actionBase.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); - List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size(), s3ObjectSummariesAfterDelete.size(), "Should be same number of files after deleting bogus path"); } @@ -107,18 +110,19 @@ public class S3BackendModuleTest extends BaseS3Test // first list the files (non-recursively) - then move one into a sub-folder, then re-list, and // // assert that we have one fewer then list again including sub-folders, and see the changed count // //////////////////////////////////////////////////////////////////////////////////////////////////// - List s3ObjectSummariesBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); - List s3ObjectSummariesInSubFolderBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, subPath, false); - List s3ObjectSummariesRecursiveBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, true); + List s3ObjectSummariesBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); + List s3ObjectSummariesInSubFolderBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, subPath, ""); + List s3ObjectSummariesRecursiveBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**"); - S3BackendModule s3BackendModule = new S3BackendModule(); - s3BackendModule.setS3Utils(getS3Utils()); + S3BackendModule s3BackendModule = new S3BackendModule(); + AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); + actionBase.setS3Utils(getS3Utils()); String key = s3ObjectSummariesBeforeMove.get(0).getKey(); - s3BackendModule.moveFile(qInstance, table, key, key.replaceFirst(TEST_FOLDER, subPath)); + actionBase.moveFile(qInstance, table, key, key.replaceFirst(TEST_FOLDER, subPath)); - List s3ObjectSummariesAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); - List s3ObjectSummariesRecursiveAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, true); - List s3ObjectSummariesInSubFolderAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, subPath, false); + List s3ObjectSummariesAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); + List s3ObjectSummariesRecursiveAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**"); + List s3ObjectSummariesInSubFolderAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, subPath, ""); Assertions.assertEquals(s3ObjectSummariesBeforeMove.size() - 1, s3ObjectSummariesAfterMove.size(), "Should be one fewer file in the non-recursive listing after moving one."); @@ -140,11 +144,12 @@ public class S3BackendModuleTest extends BaseS3Test QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3); String subPath = TEST_FOLDER + "/" + SUB_FOLDER; - S3BackendModule s3BackendModule = new S3BackendModule(); - s3BackendModule.setS3Utils(getS3Utils()); + S3BackendModule s3BackendModule = new S3BackendModule(); + AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); + actionBase.setS3Utils(getS3Utils()); Assertions.assertThrows(FilesystemException.class, () -> - s3BackendModule.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, subPath + "/" + UUID.randomUUID()) + actionBase.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, subPath + "/" + UUID.randomUUID()) ); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java index 6211d4ce..acef8ec6 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java @@ -52,7 +52,7 @@ class S3BackendMetaDataTest System.out.println(JsonUtils.prettyPrint(json)); System.out.println(json); String expectToContain = """ - {"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","backendType":"s3","name":"s3"}"""; + {"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","secretKey":null,"accessKey":null,"backendType":"s3","name":"s3","region":null}"""; assertTrue(json.contains(expectToContain)); } @@ -71,7 +71,7 @@ class S3BackendMetaDataTest QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json); assertThat(deserialized).usingRecursiveComparison() - .ignoringFields("hasBeenValidated") + .ignoringFields("hasBeenValidated") // note, this field is @JsonIgnore .isEqualTo(qInstance); } } \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/PathMatcherGlobTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/PathMatcherGlobTest.java new file mode 100644 index 00000000..2aa65293 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/PathMatcherGlobTest.java @@ -0,0 +1,73 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.utils; + + +import java.net.URI; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Verification for some of the behavior in the S3Utils - working with PathMatcher + ** globs. + *******************************************************************************/ +public class PathMatcherGlobTest +{ + + @Test + public void testPathMatcher() throws Exception + { + ///////////////////////////////////////////////////////////// + // note: must start with for both the pattern and the uri // + ///////////////////////////////////////////////////////////// + PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/*/acme/*/*.csv"); + + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/1234.csv"))), "Glob should match"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/nj/acme/20220627/1234.csv"))), "Glob should match"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220628/1234.csv"))), "Glob should match"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/12345.csv"))), "Glob should match"); + + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/beta/20220627/1234.csv"))), "Glob should not match (beta vs acme)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///something/stl/acme/20220627/1234.csv"))), "Glob should not match (wrong start path)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/csv"))), "Glob should not match (no file basename)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/1234.CSV"))), "Glob should not match (wrong case extension)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/extra/1234.csv"))), "Glob should not match (extra dir)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/extra/acme/20220627/1234.csv"))), "Glob should not match (extra dir)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/extra/stl/acme/20220627/1234.csv"))), "Glob should not match (extra dir)"); + + pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/**/acme/*/*.csv"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/extra/stl/acme/20220627/1234.csv"))), "Glob should match with extra dir"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/extra/extra2/stl/acme/20220627/1234.csv"))), "Glob should match with 2 extra dirs"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/acme/20220627/1234.csv"))), "Glob does not match with no dir for **"); + + pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/**"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/1234.csv"))), "Glob should match with extra dir"); + + pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/*"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///1234.csv"))), "Glob should match"); + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java index 0de6c5e2..ed4afbdf 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.utils; import java.io.IOException; import java.io.InputStream; +import java.util.List; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import org.apache.commons.io.IOUtils; @@ -44,15 +45,23 @@ public class S3UtilsTest extends BaseS3Test public void testListObjectsInBucketAtPath() { S3Utils s3Utils = getS3Utils(); - assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/", false).size(), "Expected # of s3 objects without subfolders"); - assertEquals(3, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/", true).size(), "Expected # of s3 objects with subfolders"); - assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "/" + TEST_FOLDER + "/", false).size(), "With leading slash"); - assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "/" + TEST_FOLDER, false).size(), "Without trailing slash"); - assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "//" + TEST_FOLDER + "//", false).size(), "With multiple leading and trailing slashes"); - assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, false).size(), "Just in the subfolder non-recursive"); - assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, true).size(), "Just in the subfolder recursive"); - assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "//" + SUB_FOLDER, true).size(), "Just in the subfolder recursive"); - assertEquals(0, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "not-a-real-path/", true).size(), "In a non-existing folder"); + assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders"); + assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders"); + assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders"); + assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders"); + assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash"); + assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash"); + assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, "").size(), "Just in the subfolder non-recursive"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, "/**").size(), "Just in the subfolder recursive"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "//" + SUB_FOLDER, "/**").size(), "Just in the subfolder recursive, multi /"); + assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "not-a-real-path/", "").size(), "In a non-existing folder"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string"); + assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively"); + assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively"); } @@ -63,12 +72,12 @@ public class S3UtilsTest extends BaseS3Test @Test public void testGetObjectAsInputStream() throws IOException { - S3Utils s3Utils = getS3Utils(); - S3ObjectSummary s3ObjectSummary = s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "test-files", true).get(0); - InputStream inputStream = s3Utils.getObjectAsInputStream(s3ObjectSummary); - String csvFromS3 = IOUtils.toString(inputStream); + S3Utils s3Utils = getS3Utils(); + List s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", ""); + S3ObjectSummary s3ObjectSummary = s3ObjectSummaries.stream().filter(o -> o.getKey().contains("1.csv")).findAny().get(); + InputStream inputStream = s3Utils.getObjectAsInputStream(s3ObjectSummary); + String csvFromS3 = IOUtils.toString(inputStream); - // todo - should check the filename somewhere, right? assertEquals(getCSVData1(), csvFromS3, "File from S3 should match expected content"); }