QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release/0.0.0'

Browse Source
This commit is contained in:
darin.kelkhoff
2022-07-01 11:29:13 -05:00
parent b3696ee558 e084f8f711
commit 54cca2c844
46 changed files with 4928 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
.circleci/config.yml
View File

@ -1,3 +1,6 @@
## Deviations from qqq-java library standard circleci config:
## - To test AWS S3, uses localstsack executor, orb, and startup step
## - This docker image doesn't have java-17, so we install (and cache) jvm-17
version: 2.1
executors:
@ -8,8 +11,18 @@ executors:
orbs:
slack: circleci/slack@4.10.1
localstack: localstack/platform@1.0
commands:
install_java17:
steps:
- run:
name: Install Java 17
command: |
sudo add-apt-repository -y ppa:openjdk-r/ppa
sudo apt install -y openjdk-17-jdk
sudo rm /etc/alternatives/java
sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java
run_maven:
parameters:
maven_subcommand:
@ -39,16 +52,20 @@ commands:
jobs:
mvn_test:
executor: java17
executor: localstack/default
steps:
- localstack/startup
- install_java17
- run_maven:
maven_subcommand: test
- slack/notify:
event: fail
mvn_deploy:
executor: java17
executor: localstack/default
steps:
- localstack/startup
- install_java17
- run_maven:
maven_subcommand: deploy
- slack/notify:

22
pom.xml
View File

@ -25,7 +25,7 @@
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-backend-module-filesystem</artifactId>
<version>0.0.0-SNAPSHOT</version>
<version>0.0.0</version>
<scm>
<connection>scm:git:git@github.com:Kingsrook/qqq-backend-module-rdbms.git</connection>
@ -51,7 +51,7 @@
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-backend-core</artifactId>
<version>0.0.0-SNAPSHOT</version>
<version>0.0.0</version>
</dependency>
<!-- 3rd party deps specifically for this module -->
@ -61,21 +61,9 @@
<version>1.12.243</version>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>1.17.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>localstack</artifactId>
<version>1.17.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<version>1.17.2</version>
<groupId>cloud.localstack</groupId>
<artifactId>localstack-utils</artifactId>
<version>0.2.20</version>
<scope>test</scope>
</dependency>

39
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java Normal file
View File

@ -0,0 +1,39 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
/*******************************************************************************
** Interface to add additional functionality commonly among the various filesystem
** module implementations.
*******************************************************************************/
public interface FilesystemBackendModuleInterface<FILE>
{
/*******************************************************************************
** For filesystem backends, get the module-specific action base-class, that helps
** with functions like listing and deleting files.
*******************************************************************************/
AbstractBaseFilesystemAction<FILE> getActionBase();
}

32
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemRecordBackendDetailFields.java Normal file
View File

@ -0,0 +1,32 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base;
/*******************************************************************************
** Define the field names (keys) to be used in the backendDetails structure
** under Records from this (these) modules.
*******************************************************************************/
public interface FilesystemRecordBackendDetailFields
{
String FULL_PATH = "fullPath";
}

294
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java Normal file
View File

@ -0,0 +1,294 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base.actions;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.NotImplementedException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Base class for all Filesystem actions across all modules.
**
** @param FILE The class that represents a file in the sub-module. e.g.,
* a java.io.File, or an S3Object.
*******************************************************************************/
public abstract class AbstractBaseFilesystemAction<FILE>
{
private static final Logger LOG = LogManager.getLogger(AbstractBaseFilesystemAction.class);
/*******************************************************************************
** List the files for a table - to be implemented in module-specific subclasses.
*******************************************************************************/
public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase);
/*******************************************************************************
** Read the contents of a file - to be implemented in module-specific subclasses.
*******************************************************************************/
public abstract InputStream readFile(FILE file) throws IOException;
/*******************************************************************************
** Write a file - to be implemented in module-specific subclasses.
*******************************************************************************/
public abstract void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException;
/*******************************************************************************
** Get a string that represents the full path to a file.
*******************************************************************************/
public abstract String getFullPathForFile(FILE file);
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
*******************************************************************************/
public abstract void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException;
/*******************************************************************************
** Move a file from a source path, to a destination path.
**
** @throws FilesystemException if the move is known to have failed
*******************************************************************************/
public abstract void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException;
/*******************************************************************************
** e.g., with a base path of /foo/
** and a table path of /bar/
** and a file at /foo/bar/baz.txt
** give us just the baz.txt part.
*******************************************************************************/
public abstract String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData sourceBackend, QTableMetaData sourceTable);
/*******************************************************************************
** Append together the backend's base path (if present), with a table's base
** path (again, if present).
*******************************************************************************/
public String getFullBasePath(QTableMetaData table, QBackendMetaData backendBase)
{
AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase);
String fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : "";
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
if(StringUtils.hasContent(tableDetails.getBasePath()))
{
fullPath += File.separatorChar + tableDetails.getBasePath();
}
fullPath += File.separatorChar;
fullPath = stripDuplicatedSlashes(fullPath);
return fullPath;
}
/*******************************************************************************
**
*******************************************************************************/
public static String stripDuplicatedSlashes(String path)
{
if(path == null)
{
return (null);
}
return (path.replaceAll("//+", "/"));
}
/*******************************************************************************
** Get the backend metaData, type-checked as the requested type.
*******************************************************************************/
protected <T extends AbstractFilesystemBackendMetaData> T getBackendMetaData(Class<T> outputClass, QBackendMetaData metaData)
{
if(!(outputClass.isInstance(metaData)))
{
throw new IllegalArgumentException("MetaData was not of expected type (was " + metaData.getClass().getSimpleName() + ")");
}
return outputClass.cast(metaData);
}
/*******************************************************************************
** Get the backendDetails out of a table, type-checked as the requested type
*******************************************************************************/
protected <T extends AbstractFilesystemTableBackendDetails> T getTableBackendDetails(Class<T> outputClass, QTableMetaData tableMetaData)
{
QTableBackendDetails tableBackendDetails = tableMetaData.getBackendDetails();
if(!(outputClass.isInstance(tableBackendDetails)))
{
throw new IllegalArgumentException("Table backend details was not of expected type (was " + tableBackendDetails.getClass().getSimpleName() + ")");
}
return outputClass.cast(tableBackendDetails);
}
/*******************************************************************************
** Generic implementation of the execute method from the QueryInterface
*******************************************************************************/
public QueryResult executeQuery(QueryRequest queryRequest) throws QException
{
preAction(queryRequest.getBackend());
try
{
QueryResult rs = new QueryResult();
List<QRecord> records = new ArrayList<>();
rs.setRecords(records);
QTableMetaData table = queryRequest.getTable();
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
List<FILE> files = listFiles(table, queryRequest.getBackend());
for(FILE file : files)
{
LOG.info("Processing file: " + getFullPathForFile(file));
switch(tableDetails.getRecordFormat())
{
case CSV:
{
String fileContents = IOUtils.toString(readFile(file));
fileContents = customizeFileContentsAfterReading(table, fileContents);
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
records.addAll(recordsInFile);
break;
}
case JSON:
{
String fileContents = IOUtils.toString(readFile(file));
fileContents = customizeFileContentsAfterReading(table, fileContents);
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
addBackendDetailsToRecords(recordsInFile, file);
records.addAll(recordsInFile);
break;
}
default:
{
throw new NotImplementedException("Filesystem record format " + tableDetails.getRecordFormat() + " is not yet implemented");
}
}
}
return rs;
}
catch(Exception e)
{
LOG.warn("Error executing query", e);
throw new QException("Error executing query", e);
}
}
/*******************************************************************************
** Add backend details to records about the file that they are in.
*******************************************************************************/
protected void addBackendDetailsToRecords(List<QRecord> recordsInFile, FILE file)
{
recordsInFile.forEach(record ->
{
record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, getFullPathForFile(file));
});
}
/*******************************************************************************
** Method that subclasses can override to add pre-action things (e.g., setting up
** s3 client).
*******************************************************************************/
public void preAction(QBackendMetaData backendMetaData)
{
/////////////////////////////////////////////////////////////////////
// noop in base class - subclasses can add functionality if needed //
/////////////////////////////////////////////////////////////////////
}
/*******************************************************************************
**
*******************************************************************************/
private String customizeFileContentsAfterReading(QTableMetaData table, String fileContents) throws QException
{
Optional<QCodeReference> optionalCustomizer = table.getCustomizer("postFileRead");
if(optionalCustomizer.isEmpty())
{
return (fileContents);
}
QCodeReference customizer = optionalCustomizer.get();
try
{
Class<?> customizerClass = Class.forName(customizer.getName());
@SuppressWarnings("unchecked")
Function<String, String> function = (Function<String, String>) customizerClass.getConstructor().newInstance();
return function.apply(fileContents);
}
catch(Exception e)
{
throw (new QException("Error customizing file contents", e));
}
}
}

80
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java Normal file
View File

@ -0,0 +1,80 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
/*******************************************************************************
** Base class for all BackendMetaData for all filesystem-style backend modules.
*******************************************************************************/
public class AbstractFilesystemBackendMetaData extends QBackendMetaData
{
private String basePath;
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public AbstractFilesystemBackendMetaData()
{
super();
}
/*******************************************************************************
** Getter for basePath
**
*******************************************************************************/
public String getBasePath()
{
return (basePath);
}
/*******************************************************************************
** Setter for basePath
**
*******************************************************************************/
public void setBasePath(String basePath)
{
this.basePath = basePath;
}
/*******************************************************************************
** Fluent setter for basePath
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends AbstractFilesystemBackendMetaData> T withBasePath(String basePath)
{
this.basePath = basePath;
return (T) this;
}
}

178
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java Normal file
View File

@ -0,0 +1,178 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
/*******************************************************************************
** Extension of QTableBackendDetails, with details specific to a Filesystem table.
*******************************************************************************/
public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
{
private String basePath;
private String glob;
private RecordFormat recordFormat;
private Cardinality cardinality;
/*******************************************************************************
** Getter for basePath
**
*******************************************************************************/
public String getBasePath()
{
return basePath;
}
/*******************************************************************************
** Setter for basePath
**
*******************************************************************************/
public void setBasePath(String basePath)
{
this.basePath = basePath;
}
/*******************************************************************************
** Fluent Setter for basePath
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends AbstractFilesystemTableBackendDetails> T withBasePath(String basePath)
{
this.basePath = basePath;
return (T) this;
}
/*******************************************************************************
** Getter for glob
**
*******************************************************************************/
public String getGlob()
{
return glob;
}
/*******************************************************************************
** Setter for glob
**
*******************************************************************************/
public void setGlob(String glob)
{
this.glob = glob;
}
/*******************************************************************************
** Fluent Setter for glob
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends AbstractFilesystemTableBackendDetails> T withGlob(String glob)
{
this.glob = glob;
return (T) this;
}
/*******************************************************************************
** Getter for recordFormat
**
*******************************************************************************/
public RecordFormat getRecordFormat()
{
return recordFormat;
}
/*******************************************************************************
** Setter for recordFormat
**
*******************************************************************************/
public void setRecordFormat(RecordFormat recordFormat)
{
this.recordFormat = recordFormat;
}
/*******************************************************************************
** Fluent Setter for recordFormat
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends AbstractFilesystemTableBackendDetails> T withRecordFormat(RecordFormat recordFormat)
{
this.recordFormat = recordFormat;
return ((T) this);
}
/*******************************************************************************
** Getter for cardinality
**
*******************************************************************************/
public Cardinality getCardinality()
{
return cardinality;
}
/*******************************************************************************
** Setter for cardinality
**
*******************************************************************************/
public void setCardinality(Cardinality cardinality)
{
this.cardinality = cardinality;
}
/*******************************************************************************
** Fluent Setter for cardinality
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends AbstractFilesystemTableBackendDetails> T withCardinality(Cardinality cardinality)
{
this.cardinality = cardinality;
return ((T) this);
}
}

33
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/Cardinality.java Normal file
View File

@ -0,0 +1,33 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
/*******************************************************************************
** Mode for filesystem backends: are all records in a single file, or are there
** many files?
*******************************************************************************/
public enum Cardinality
{
ONE,
MANY
}

34
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/RecordFormat.java Normal file
View File

@ -0,0 +1,34 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
/*******************************************************************************
** How are records stored in the files in a filesystem backend? CSV, JSON,
** future types may XML, or more exotic ones, like "BINARY" or "TEXT" (e.g., 1
** record and 1 field per-file)
*******************************************************************************/
public enum RecordFormat
{
CSV,
JSON
}

52
src/main/java/com/kingsrook/qqq/backend/module/filesystem/exceptions/FilesystemException.java Normal file
View File

@ -0,0 +1,52 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.exceptions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
/*******************************************************************************
** Checked exception to be thrown from actions within this module.
*******************************************************************************/
public class FilesystemException extends QException
{
/*******************************************************************************
** Constructor of message
**
*******************************************************************************/
public FilesystemException(String message)
{
super(message);
}
/*******************************************************************************
** Constructor of message & cause
**
*******************************************************************************/
public FilesystemException(String message, Throwable cause)
{
super(message, cause);
}
}

142
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java Normal file
View File

@ -0,0 +1,142 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local;
import java.io.File;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemUpdateAction;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** QQQ Backend module for working with (local) Filesystems.
*******************************************************************************/
public class FilesystemBackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface
{
private static final Logger LOG = LogManager.getLogger(FilesystemBackendModule.class);
/*******************************************************************************
** For filesystem backends, get the module-specific action base-class, that helps
** with functions like listing and deleting files.
*******************************************************************************/
@Override
public AbstractBaseFilesystemAction<File> getActionBase()
{
return (new AbstractFilesystemAction());
}
/*******************************************************************************
** Method where a backend module must be able to provide its type (name).
*******************************************************************************/
@Override
public String getBackendType()
{
return ("filesystem");
}
/*******************************************************************************
** Method to identify the class used for backend meta data for this module.
*******************************************************************************/
@Override
public Class<? extends QBackendMetaData> getBackendMetaDataClass()
{
return (FilesystemBackendMetaData.class);
}
/*******************************************************************************
** Method to identify the class used for table-backend details for this module.
*******************************************************************************/
@Override
public Class<? extends QTableBackendDetails> getTableBackendDetailsClass()
{
return FilesystemTableBackendDetails.class;
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QueryInterface getQueryInterface()
{
return new FilesystemQueryAction();
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public InsertInterface getInsertInterface()
{
return (new FilesystemInsertAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public UpdateInterface getUpdateInterface()
{
return (new FilesystemUpdateAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public DeleteInterface getDeleteInterface()
{
return (new FilesystemDeleteAction());
}
}

186
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java Normal file
View File

@ -0,0 +1,186 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Base class for all (local) Filesystem actions
*******************************************************************************/
public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
{
private static final Logger LOG = LogManager.getLogger(AbstractFilesystemAction.class);
/*******************************************************************************
** List the files for this table.
*******************************************************************************/
@Override
public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase)
{
// todo - needs rewritten to do globbing...
String fullPath = getFullBasePath(table, backendBase);
File directory = new File(fullPath);
File[] files = directory.listFiles();
if(files == null)
{
return Collections.emptyList();
}
return (Arrays.stream(files).filter(File::isFile).toList());
}
/*******************************************************************************
** Read the contents of a file.
*******************************************************************************/
@Override
public InputStream readFile(File file) throws IOException
{
return (new FileInputStream(file));
}
/*******************************************************************************
** Write a file - to be implemented in module-specific subclasses.
*******************************************************************************/
@Override
public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
{
FileUtils.writeByteArrayToFile(new File(path), contents);
}
/*******************************************************************************
** Get a string that represents the full path to a file.
*******************************************************************************/
@Override
public String getFullPathForFile(File file)
{
return (file.getAbsolutePath());
}
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
*******************************************************************************/
@Override
public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
{
File file = new File(fileReference);
if(!file.exists())
{
//////////////////////////////////////////////////////////////////////////////////////////////
// if the file doesn't exist, just exit with noop. don't throw an error - that should only //
// happen if the "contract" of the method is broken, and the file still exists //
//////////////////////////////////////////////////////////////////////////////////////////////
LOG.debug("Not deleting file [{}], because it does not exist.", file);
return;
}
if(!file.delete())
{
throw (new FilesystemException("Failed to delete file: " + fileReference));
}
}
/*******************************************************************************
** Move a file from a source path, to a destination path.
**
** @param destination assumed to be a file path - not a directory
** @throws FilesystemException if the delete is known to have failed
*******************************************************************************/
@Override
public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException
{
File sourceFile = new File(source);
File destinationFile = new File(destination);
File destinationParent = destinationFile.getParentFile();
if(!sourceFile.exists())
{
throw (new FilesystemException("Cannot move file " + source + ", as it does not exist."));
}
//////////////////////////////////////////////////////////////////////////////////////
// if the destination folder doesn't exist, try to make it - and fail if that fails //
//////////////////////////////////////////////////////////////////////////////////////
if(!destinationParent.exists())
{
LOG.debug("Making destination directory {} for move", destinationParent.getAbsolutePath());
if(!destinationParent.mkdirs())
{
throw (new FilesystemException("Failed to make destination directory " + destinationParent.getAbsolutePath() + " to move " + source + " into."));
}
}
if(!sourceFile.renameTo(destinationFile))
{
throw (new FilesystemException("Failed to move (rename) file " + source + " to " + destination));
}
}
/*******************************************************************************
** e.g., with a base path of /foo/
** and a table path of /bar/
** and a file at /foo/bar/baz.txt
** give us just the baz.txt part.
*******************************************************************************/
@Override
public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table)
{
String tablePath = getFullBasePath(table, backend);
String strippedPath = filePath.replaceFirst(".*" + tablePath, "");
return (strippedPath);
}
}

60
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java Normal file
View File

@ -0,0 +1,60 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class FilesystemDeleteAction implements DeleteInterface
{
/*******************************************************************************
**
*******************************************************************************/
public DeleteResult execute(DeleteRequest deleteRequest) throws QException
{
throw new NotImplementedException("Filesystem delete not implemented");
/*
try
{
DeleteResult rs = new DeleteResult();
QTableMetaData table = deleteRequest.getTable();
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing delete: " + e.getMessage(), e);
}
*/
}
}

62
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java Normal file
View File

@ -0,0 +1,62 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class FilesystemInsertAction implements InsertInterface
{
/*******************************************************************************
**
*******************************************************************************/
public InsertResult execute(InsertRequest insertRequest) throws QException
{
throw new NotImplementedException("Filesystem insert not implemented");
/*
try
{
InsertResult rs = new InsertResult();
QTableMetaData table = insertRequest.getTable();
List<QRecord> recordsWithStatus = new ArrayList<>();
rs.setRecords(recordsWithStatus);
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
*/
}
}

45
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryAction.java Normal file
View File

@ -0,0 +1,45 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
/*******************************************************************************
**
*******************************************************************************/
public class FilesystemQueryAction extends AbstractFilesystemAction implements QueryInterface
{
/*******************************************************************************
**
*******************************************************************************/
public QueryResult execute(QueryRequest queryRequest) throws QException
{
return executeQuery(queryRequest);
}
}

63
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java Normal file
View File

@ -0,0 +1,63 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class FilesystemUpdateAction implements UpdateInterface
{
/*******************************************************************************
**
*******************************************************************************/
public UpdateResult execute(UpdateRequest updateRequest) throws QException
{
throw new NotImplementedException("Filesystem update not implemented");
/*
try
{
UpdateResult rs = new UpdateResult();
QTableMetaData table = updateRequest.getTable();
List<QRecord> records = new ArrayList<>();
rs.setRecords(records);
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing update: " + e.getMessage(), e);
}
*/
}
}

45
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java Normal file
View File

@ -0,0 +1,45 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule;
/*******************************************************************************
** (local) Filesystem backend meta data.
*******************************************************************************/
public class FilesystemBackendMetaData extends AbstractFilesystemBackendMetaData
{
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public FilesystemBackendMetaData()
{
super();
setBackendType(FilesystemBackendModule.class);
}
}

44
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemTableBackendDetails.java Normal file
View File

@ -0,0 +1,44 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule;
/*******************************************************************************
** (local) Filesystem specific Extension of QTableBackendDetails
*******************************************************************************/
public class FilesystemTableBackendDetails extends AbstractFilesystemTableBackendDetails
{
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public FilesystemTableBackendDetails()
{
super();
setBackendType(FilesystemBackendModule.class);
}
}

135
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java Normal file
View File

@ -0,0 +1,135 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
import java.io.File;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Function body for performing the Cleanup step of a basic ETL process - e.g.,
** after the loading, delete or move the processed file(s).
*******************************************************************************/
public class BasicETLCleanupSourceFilesFunction implements FunctionBody
{
private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesFunction.class);
public static final String FIELD_MOVE_OR_DELETE = "moveOrDelete";
public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves";
public static final String VALUE_MOVE = "move";
public static final String VALUE_DELETE = "delete";
public static final String FUNCTION_NAME = "cleanupSourceFiles";
/*******************************************************************************
** Execute the function - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
{
String sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
QTableMetaData table = runFunctionRequest.getInstance().getTable(sourceTableName);
QBackendMetaData backend = runFunctionRequest.getInstance().getBackendForTable(sourceTableName);
QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend);
if(!(module instanceof FilesystemBackendModuleInterface filesystemModule))
{
throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function."));
}
AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase();
actionBase.preAction(backend);
String sourceFilePaths = runFunctionRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS);
if(!StringUtils.hasContent(sourceFilePaths))
{
LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS + "]");
return;
}
String[] sourceFiles = sourceFilePaths.split(",");
for(String sourceFile : sourceFiles)
{
String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE);
if(VALUE_DELETE.equals(moveOrDelete))
{
actionBase.deleteFile(runFunctionRequest.getInstance(), table, sourceFile);
}
else if(VALUE_MOVE.equals(moveOrDelete))
{
String destinationForMoves = runFunctionRequest.getValueString(FIELD_DESTINATION_FOR_MOVES);
if(!StringUtils.hasContent(destinationForMoves))
{
throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value."));
}
String filePathWithoutBase = actionBase.stripBackendAndTableBasePathsFromFileName(sourceFile, backend, table);
String destinationPath = destinationForMoves + File.separator + filePathWithoutBase;
actionBase.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath);
}
else
{
throw (new QException("Unexpected value [" + moveOrDelete + "] for field [" + FIELD_MOVE_OR_DELETE + "]. "
+ "Must be either [" + VALUE_MOVE + "] or [" + VALUE_DELETE + "]."));
}
}
}
/*******************************************************************************
** define the metaData that describes this function
*******************************************************************************/
public QFunctionMetaData defineFunctionMetaData()
{
return (new QFunctionMetaData()
.withName(FUNCTION_NAME)
.withCode(new QCodeReference()
.withName(this.getClass().getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.FUNCTION))
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData("moveOrDelete", QFieldType.STRING))
.addField(new QFieldMetaData("destinationForMoves", QFieldType.STRING))));
}
}

84
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java Normal file
View File

@ -0,0 +1,84 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
/*******************************************************************************
** Function body for collecting the file names that were discovered in the
** Extract step. These will be lost during the transform, so we capture them here,
** so that our Clean function can move or delete them.
**
** TODO - need unit test!!
*******************************************************************************/
public class BasicETLCollectSourceFileNamesFunction implements FunctionBody
{
public static final String FUNCTION_NAME = "collectSourceFileNames";
public static final String FIELD_SOURCE_FILE_PATHS = "sourceFilePaths";
/*******************************************************************************
** Execute the function - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
{
Set<String> sourceFiles = runFunctionRequest.getRecords().stream()
.map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))
.collect(Collectors.toSet());
runFunctionResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles));
}
/*******************************************************************************
** define the metaData that describes this function
*******************************************************************************/
public QFunctionMetaData defineFunctionMetaData()
{
return (new QFunctionMetaData()
.withName(FUNCTION_NAME)
.withCode(new QCodeReference()
.withName(this.getClass().getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.FUNCTION))
.withOutputMetaData(new QFunctionOutputMetaData()
.addField(new QFieldMetaData(FIELD_SOURCE_FILE_PATHS, QFieldType.STRING))));
}
}

138
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java Normal file
View File

@ -0,0 +1,138 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync;
import java.io.File;
import java.io.InputStream;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Function body for collecting the file names that were discovered in the
** Extract step. These will be lost during the transform, so we capture them here,
** so that our Clean function can move or delete them.
**
*******************************************************************************/
public class FilesystemSyncFunction implements FunctionBody
{
private static final Logger LOG = LogManager.getLogger(FilesystemSyncFunction.class);
public static final String FUNCTION_NAME = "sync";
/*******************************************************************************
** Execute the function - using the request as input, and the result as output.
*******************************************************************************/
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
{
QTableMetaData sourceTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
QTableMetaData archiveTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName());
FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase();
sourceActionBase.preAction(sourceBackend);
Map<String, Object> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName());
FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase();
archiveActionBase.preAction(archiveBackend);
Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet();
QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName());
FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase();
processingActionBase.preAction(processingBackend);
Integer maxFilesToSync = runFunctionRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
int syncedFileCount = 0;
for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet())
{
try
{
String sourceFileName = sourceEntry.getKey();
if(!archiveFiles.contains(sourceFileName))
{
LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]");
InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue());
byte[] bytes = inputStream.readAllBytes();
String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
syncedFileCount++;
if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)
{
LOG.info("Breaking after syncing " + syncedFileCount + " files");
break;
}
}
}
catch(Exception e)
{
LOG.error("Error processing file: " + sourceEntry, e);
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private Map<String, Object> getFileNames(AbstractBaseFilesystemAction actionBase, QTableMetaData table, QBackendMetaData backend)
{
List<Object> files = actionBase.listFiles(table, backend);
Map<String, Object> rs = new LinkedHashMap<>();
for(Object file : files)
{
String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table);
rs.put(fileName, file);
}
return (rs);
}
}

83
src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java Normal file
View File

@ -0,0 +1,83 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
/*******************************************************************************
** Definition for Filesystem sync process.
**
** Job is to:
** - list all files in the source table.
** - list all files in the archive table.
** - if any files exist in the source, but not in the archive, then:
** - copy the file to both the archive and the processing table.
**
** The maxFilesToArchive field can be used to only sync up to that many files
** (an help an initial sync, if you want to do it in smaller batches)
**
** The idea being, that the source is read-only, and we want to move files out of
** processing after they've been processed - and the archive is what we can have
** in-between the two.
*******************************************************************************/
public class FilesystemSyncProcess
{
public static final String PROCESS_NAME = "filesystem.sync";
public static final String FIELD_SOURCE_TABLE = "sourceTable";
public static final String FIELD_ARCHIVE_TABLE = "archiveTable";
public static final String FIELD_PROCESSING_TABLE = "processingTable";
public static final String FIELD_MAX_FILES_TO_ARCHIVE = "maxFilesToArchive";
/*******************************************************************************
**
*******************************************************************************/
public QProcessMetaData defineProcessMetaData()
{
QFunctionMetaData syncFunction = new QFunctionMetaData()
.withName(FilesystemSyncFunction.FUNCTION_NAME)
.withCode(new QCodeReference()
.withName(FilesystemSyncFunction.class.getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.FUNCTION))
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING))
.addField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING))
.addField(new QFieldMetaData(FIELD_MAX_FILES_TO_ARCHIVE, QFieldType.INTEGER).withDefaultValue(Integer.MAX_VALUE))
.addField(new QFieldMetaData(FIELD_PROCESSING_TABLE, QFieldType.STRING)));
return new QProcessMetaData()
.withName(PROCESS_NAME)
.addFunction(syncFunction);
}
}

138
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java Normal file
View File

@ -0,0 +1,138 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
/*******************************************************************************
** QQQ Backend module for working with AWS S3 filesystems
*******************************************************************************/
public class S3BackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface
{
/*******************************************************************************
** For filesystem backends, get the module-specific action base-class, that helps
** with functions like listing and deleting files.
*******************************************************************************/
@Override
public AbstractBaseFilesystemAction<S3ObjectSummary> getActionBase()
{
return (new AbstractS3Action());
}
/*******************************************************************************
** Method where a backend module must be able to provide its type (name).
*******************************************************************************/
@Override
public String getBackendType()
{
return ("s3");
}
/*******************************************************************************
** Method to identify the class used for backend meta data for this module.
*******************************************************************************/
@Override
public Class<? extends QBackendMetaData> getBackendMetaDataClass()
{
return (S3BackendMetaData.class);
}
/*******************************************************************************
** Method to identify the class used for table-backend details for this module.
*******************************************************************************/
@Override
public Class<? extends QTableBackendDetails> getTableBackendDetailsClass()
{
return (S3TableBackendDetails.class);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QueryInterface getQueryInterface()
{
return new S3QueryAction();
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public InsertInterface getInsertInterface()
{
return (new S3InsertAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public UpdateInterface getUpdateInterface()
{
return (new S3UpdateAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public DeleteInterface getDeleteInterface()
{
return (new S3DeleteAction());
}
}

237
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java Normal file
View File

@ -0,0 +1,237 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Base class for all S3 filesystem actions
*******************************************************************************/
public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSummary>
{
private static final Logger LOG = LogManager.getLogger(AbstractS3Action.class);
private S3Utils s3Utils;
/*******************************************************************************
** Setup the s3 utils object to be used for this action.
*******************************************************************************/
@Override
public void preAction(QBackendMetaData backendMetaData)
{
super.preAction(backendMetaData);
if(s3Utils != null)
{
LOG.debug("s3Utils object is already set - not re-setting it.");
return;
}
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendMetaData);
AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard();
if(StringUtils.hasContent(s3BackendMetaData.getAccessKey()) && StringUtils.hasContent(s3BackendMetaData.getSecretKey()))
{
BasicAWSCredentials credentials = new BasicAWSCredentials(s3BackendMetaData.getAccessKey(), s3BackendMetaData.getSecretKey());
clientBuilder.setCredentials(new AWSStaticCredentialsProvider(credentials));
}
if(StringUtils.hasContent(s3BackendMetaData.getRegion()))
{
clientBuilder.setRegion(s3BackendMetaData.getRegion());
}
s3Utils = new S3Utils();
s3Utils.setAmazonS3(clientBuilder.build());
}
/*******************************************************************************
** Set the S3Utils object.
*******************************************************************************/
public void setS3Utils(S3Utils s3Utils)
{
this.s3Utils = s3Utils;
}
/*******************************************************************************
** Internal accessor for the s3Utils object - should always use this, not the field.
*******************************************************************************/
private S3Utils getS3Utils()
{
if(s3Utils == null)
{
s3Utils = new S3Utils();
}
return s3Utils;
}
/*******************************************************************************
** List the files for a table.
*******************************************************************************/
@Override
public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase)
{
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase);
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
String fullPath = getFullBasePath(table, backendBase);
String bucketName = s3BackendMetaData.getBucketName();
String glob = tableDetails.getGlob();
////////////////////////////////////////////////////////////////////
// todo - look at metadata to configure the s3 client here? //
////////////////////////////////////////////////////////////////////
return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob);
}
/*******************************************************************************
** Read the contents of a file.
*******************************************************************************/
@Override
public InputStream readFile(S3ObjectSummary s3ObjectSummary) throws IOException
{
return (getS3Utils().getObjectAsInputStream(s3ObjectSummary));
}
/*******************************************************************************
** Write a file - to be implemented in module-specific subclasses.
*******************************************************************************/
@Override
public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException
{
path = stripLeadingSlash(stripDuplicatedSlashes(path));
String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName();
getS3Utils().writeFile(bucketName, path, contents);
}
/*******************************************************************************
**
*******************************************************************************/
private String stripLeadingSlash(String path)
{
if(path == null)
{
return (null);
}
return (path.replaceFirst("^/+", ""));
}
/*******************************************************************************
** Get a string that represents the full path to a file.
*******************************************************************************/
@Override
public String getFullPathForFile(S3ObjectSummary s3ObjectSummary)
{
return (s3ObjectSummary.getKey());
}
/*******************************************************************************
** e.g., with a base path of /foo/
** and a table path of /bar/
** and a file at /foo/bar/baz.txt
** give us just the baz.txt part.
*******************************************************************************/
@Override
public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table)
{
String tablePath = getFullBasePath(table, backend);
String strippedPath = filePath.replaceFirst("^/*" + stripLeadingSlash(tablePath), "");
return (strippedPath);
}
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
*******************************************************************************/
@Override
public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
{
QBackendMetaData backend = instance.getBackend(table.getBackendName());
String bucketName = ((S3BackendMetaData) backend).getBucketName();
getS3Utils().deleteObject(bucketName, fileReference);
}
/*******************************************************************************
** In contrast with the DeleteAction, which deletes RECORDS - this is a
** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE
** e.g., for post-ETL.
**
** @param destination assumed to be a file path - not a directory
** @throws FilesystemException if the move is known to have failed
*******************************************************************************/
@Override
public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException
{
QBackendMetaData backend = instance.getBackend(table.getBackendName());
String bucketName = ((S3BackendMetaData) backend).getBucketName();
destination = stripLeadingSlash(stripDuplicatedSlashes(destination));
getS3Utils().moveObject(bucketName, source, destination);
}
}

60
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java Normal file
View File

@ -0,0 +1,60 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class S3DeleteAction implements DeleteInterface
{
/*******************************************************************************
**
*******************************************************************************/
public DeleteResult execute(DeleteRequest deleteRequest) throws QException
{
throw new NotImplementedException("S3 delete not implemented");
/*
try
{
DeleteResult rs = new DeleteResult();
QTableMetaData table = deleteRequest.getTable();
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing delete: " + e.getMessage(), e);
}
*/
}
}

63
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java Normal file
View File

@ -0,0 +1,63 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class S3InsertAction implements InsertInterface
{
/*******************************************************************************
**
*******************************************************************************/
public InsertResult execute(InsertRequest insertRequest) throws QException
{
throw new NotImplementedException("S3 insert not implemented");
/*
try
{
InsertResult rs = new InsertResult();
QTableMetaData table = insertRequest.getTable();
List<QRecord> recordsWithStatus = new ArrayList<>();
rs.setRecords(recordsWithStatus);
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
*/
}
}

45
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryAction.java Normal file
View File

@ -0,0 +1,45 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
/*******************************************************************************
**
*******************************************************************************/
public class S3QueryAction extends AbstractS3Action implements QueryInterface
{
/*******************************************************************************
**
*******************************************************************************/
public QueryResult execute(QueryRequest queryRequest) throws QException
{
return (super.executeQuery(queryRequest));
}
}

62
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java Normal file
View File

@ -0,0 +1,62 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class S3UpdateAction implements UpdateInterface
{
/*******************************************************************************
**
*******************************************************************************/
public UpdateResult execute(UpdateRequest updateRequest) throws QException
{
throw new NotImplementedException("S3 update not implemented");
/*
try
{
UpdateResult rs = new UpdateResult();
QTableMetaData table = updateRequest.getTable();
List<QRecord> records = new ArrayList<>();
rs.setRecords(records);
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing update: " + e.getMessage(), e);
}
*/
}
}

207
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java Normal file
View File

@ -0,0 +1,207 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
/*******************************************************************************
** S3 backend meta data.
*******************************************************************************/
public class S3BackendMetaData extends AbstractFilesystemBackendMetaData
{
private String bucketName;
private String accessKey;
private String secretKey;
private String region;
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public S3BackendMetaData()
{
super();
setBackendType(S3BackendModule.class);
}
/*******************************************************************************
** Getter for bucketName
**
*******************************************************************************/
public String getBucketName()
{
return bucketName;
}
/*******************************************************************************
** Setter for bucketName
**
*******************************************************************************/
public void setBucketName(String bucketName)
{
this.bucketName = bucketName;
}
/*******************************************************************************
** Fluent setter for bucketName
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends S3BackendMetaData> T withBucketName(String bucketName)
{
this.bucketName = bucketName;
return (T) this;
}
/*******************************************************************************
** Getter for accessKey
**
*******************************************************************************/
public String getAccessKey()
{
return accessKey;
}
/*******************************************************************************
** Setter for accessKey
**
*******************************************************************************/
public void setAccessKey(String accessKey)
{
this.accessKey = accessKey;
}
/*******************************************************************************
** Fluent setter for accessKey
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends S3BackendMetaData> T withAccessKey(String accessKey)
{
this.accessKey = accessKey;
return (T) this;
}
/*******************************************************************************
** Getter for secretKey
**
*******************************************************************************/
public String getSecretKey()
{
return secretKey;
}
/*******************************************************************************
** Setter for secretKey
**
*******************************************************************************/
public void setSecretKey(String secretKey)
{
this.secretKey = secretKey;
}
/*******************************************************************************
** Fluent setter for secretKey
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends S3BackendMetaData> T withSecretKey(String secretKey)
{
this.secretKey = secretKey;
return (T) this;
}
/*******************************************************************************
** Getter for region
**
*******************************************************************************/
public String getRegion()
{
return region;
}
/*******************************************************************************
** Setter for region
**
*******************************************************************************/
public void setRegion(String region)
{
this.region = region;
}
/*******************************************************************************
** Fluent setter for region
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends S3BackendMetaData> T withRegion(String region)
{
this.region = region;
return (T) this;
}
/*******************************************************************************
** Called by the QInstanceEnricher - to do backend-type-specific enrichments.
** Original use case is: reading secrets into fields (e.g., passwords).
** TODO - migrate to use @InterpretableFields (and complete that impl on core side)
*******************************************************************************/
@Override
public void enrich()
{
super.enrich();
QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
accessKey = interpreter.interpret(accessKey);
secretKey = interpreter.interpret(secretKey);
}
}

46
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3TableBackendDetails.java Normal file
View File

@ -0,0 +1,46 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
/*******************************************************************************
** S3 specific Extension of QTableBackendDetails
*******************************************************************************/
public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails
{
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public S3TableBackendDetails()
{
super();
setBackendType(S3BackendModule.class);
}
}

249
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java Normal file
View File

@ -0,0 +1,249 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.net.URI;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.util.ArrayList;
import java.util.List;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Utility methods for working with AWS S3.
**
** Note: May need a constructor (or similar) in the future that takes the
** S3BackendMetaData - e.g., if we need some metaData to construct the AmazonS3
** (api client) object, such as region, or authentication.
*******************************************************************************/
public class S3Utils
{
private static final Logger LOG = LogManager.getLogger(S3Utils.class);
private AmazonS3 amazonS3;
/*******************************************************************************
** List the objects in an S3 bucket matching a glob, per:
** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
*******************************************************************************/
public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob)
{
//////////////////////////////////////////////////////////////////////////////////////////////////
// s3 list requests find nothing if the path starts with a /, so strip away any leading slashes //
// also strip away trailing /'s, for consistent known paths. //
// also normalize any duplicated /'s to a single /. //
//////////////////////////////////////////////////////////////////////////////////////////////////
path = path.replaceFirst("^/+", "").replaceFirst("/+$", "").replaceAll("//+", "/");
String prefix = path;
// todo - maybe this is some error - that the user put a * in the path instead of the glob?
if(prefix.indexOf('*') > -1)
{
prefix = prefix.substring(0, prefix.indexOf('*'));
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob. //
// That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. //
// Anyway, add a trailing /* to globs with no wildcards (or just a '*' if it's a request for the root ("")) //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(glob == null)
{
glob = "";
}
if(!glob.contains("*"))
{
if(glob.equals(""))
{
glob += "*";
}
else
{
glob += "/*";
}
}
String pathMatcherArg = AbstractFilesystemAction.stripDuplicatedSlashes("glob:/" + path + "/" + glob);
PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher(pathMatcherArg);
ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request()
.withBucketName(bucketName)
.withPrefix(prefix);
ListObjectsV2Result listObjectsV2Result = null;
List<S3ObjectSummary> rs = new ArrayList<>();
do
{
if(listObjectsV2Result != null)
{
listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken());
}
LOG.info("Listing bucket=" + bucketName + ", path=" + path);
listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request);
//////////////////////////////////
// put files in the result list //
//////////////////////////////////
for(S3ObjectSummary objectSummary : listObjectsV2Result.getObjectSummaries())
{
String key = objectSummary.getKey();
//////////////////////////////////////////////////////////////////////////////////////////////////////
// it looks like keys in s3 can have duplicated /'s - so normalize those, to create a "sane" result //
//////////////////////////////////////////////////////////////////////////////////////////////////////
key = key.replaceAll("//+", "/");
////////////////////////////////////////////////////////////////////////////////
// always skip folders //
// this seemed to fire when it was first written, but not in our unit tests - //
// is this a difference with real s3 vs. localstack possibly? //
////////////////////////////////////////////////////////////////////////////////
if(key.endsWith("/"))
{
LOG.debug("Skipping file [{}] because it is a folder", key);
continue;
}
///////////////////////////////////////////
// skip files that do not match the glob //
///////////////////////////////////////////
if(!pathMatcher.matches(Path.of(URI.create("file:///" + key))))
{
LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob);
continue;
}
rs.add(objectSummary);
}
}
while(listObjectsV2Result.isTruncated());
return rs;
}
/*******************************************************************************
** Get the contents (as an InputStream) for an object in s3
*******************************************************************************/
public InputStream getObjectAsInputStream(S3ObjectSummary s3ObjectSummary)
{
return getAmazonS3().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()).getObjectContent();
}
/*******************************************************************************
** Write a file
*******************************************************************************/
public void writeFile(String bucket, String key, byte[] contents)
{
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentLength(contents.length);
getAmazonS3().putObject(bucket, key, new ByteArrayInputStream(contents), objectMetadata);
}
/*******************************************************************************
** Delete an object (file) from a bucket
*******************************************************************************/
public void deleteObject(String bucketName, String key) throws FilesystemException
{
//////////////////////////////////////////////////////////////////////////////////////////////
// note, aws s3 api does not appear to have any way to check the success or failure here... //
//////////////////////////////////////////////////////////////////////////////////////////////
try
{
getAmazonS3().deleteObject(bucketName, key);
}
catch(Exception e)
{
throw (new FilesystemException("Error deleting s3 object " + key + " in bucket " + bucketName, e));
}
}
/*******************************************************************************
** Move an object (file) within a bucket
*******************************************************************************/
public void moveObject(String bucketName, String source, String destination) throws FilesystemException
{
//////////////////////////////////////////////////////////////////////////////////////////////
// note, aws s3 api does not appear to have any way to check the success or failure here... //
//////////////////////////////////////////////////////////////////////////////////////////////
try
{
getAmazonS3().copyObject(bucketName, source, bucketName, destination);
getAmazonS3().deleteObject(bucketName, source);
}
catch(Exception e)
{
throw (new FilesystemException("Error moving s3 object " + source + " to " + destination + " in bucket " + bucketName, e));
}
}
/*******************************************************************************
** Setter for AmazonS3 client object.
*******************************************************************************/
public void setAmazonS3(AmazonS3 amazonS3)
{
this.amazonS3 = amazonS3;
}
/*******************************************************************************
** Getter for AmazonS3 client object.
*******************************************************************************/
public AmazonS3 getAmazonS3()
{
if(amazonS3 == null)
{
// TODO - get this (and other props?) from backend meta data
amazonS3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build();
}
return amazonS3;
}
}

227
src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java Normal file
View File

@ -0,0 +1,227 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem;
import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.mock.MockAuthenticationModule;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import org.apache.commons.io.FileUtils;
/*******************************************************************************
** Utility methods for Filesystem module tests
*******************************************************************************/
public class TestUtils
{
public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
public static final String BACKEND_NAME_S3 = "s3";
public static final String TABLE_NAME_PERSON_LOCAL_FS = "person";
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
///////////////////////////////////////////////////////////////////
// shouldn't be accessed directly, as we append a counter to it. //
///////////////////////////////////////////////////////////////////
public static final String BASE_PATH = "/tmp/filesystem-tests";
//////////////////////////////////////////////////////////////////////////////
// Used to make each test method have a unique folder path, more or less... //
// See methods that work with it. //
//////////////////////////////////////////////////////////////////////////////
private static int testInstanceCounter = 0;
/*******************************************************************************
** Meant to be called in a @BeforeEach - increment an internal counter that will
** give us a unique directory name for each test method.
*******************************************************************************/
public static void increaseTestInstanceCounter()
{
testInstanceCounter++;
}
/*******************************************************************************
** Reset the counter to 0 (e.g., to let some tests have a known value).
*******************************************************************************/
public static void resetTestInstanceCounter()
{
testInstanceCounter = 0;
}
/*******************************************************************************
** Get the current value of the testInstanceCounter. See {@link #increaseTestInstanceCounter()}
*******************************************************************************/
public static int getTestInstanceCounter()
{
return (testInstanceCounter);
}
/*******************************************************************************
** Meant to run both after and before test methods - makes sure the file system
** is empty for the path under the instance.
*******************************************************************************/
public static void cleanInstanceFiles() throws IOException
{
FileUtils.deleteDirectory(new File(BASE_PATH + File.separator + testInstanceCounter));
}
/*******************************************************************************
**
*******************************************************************************/
public static QInstance defineInstance() throws QInstanceValidationException
{
QInstance qInstance = new QInstance();
qInstance.setAuthentication(defineAuthentication());
qInstance.addBackend(defineLocalFilesystemBackend());
qInstance.addTable(defineLocalFilesystemJSONPersonTable());
qInstance.addBackend(defineS3Backend());
qInstance.addTable(defineS3CSVPersonTable());
new QInstanceValidator().validate(qInstance);
return (qInstance);
}
/*******************************************************************************
** Define the authentication used in standard tests - using 'mock' type.
**
*******************************************************************************/
public static QAuthenticationMetaData defineAuthentication()
{
return new QAuthenticationMetaData()
.withName("mock")
.withType("mock");
}
/*******************************************************************************
**
*******************************************************************************/
public static FilesystemBackendMetaData defineLocalFilesystemBackend()
{
return (new FilesystemBackendMetaData()
.withBasePath(BASE_PATH + File.separator + testInstanceCounter)
.withName(BACKEND_NAME_LOCAL_FS));
}
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineLocalFilesystemJSONPersonTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_PERSON_LOCAL_FS)
.withLabel("Person")
.withBackendName(defineLocalFilesystemBackend().getName())
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"))
.withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"))
.withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"))
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
.withField(new QFieldMetaData("email", QFieldType.STRING))
.withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath("persons")
.withRecordFormat(RecordFormat.JSON)
.withCardinality(Cardinality.MANY)
);
}
/*******************************************************************************
**
*******************************************************************************/
public static S3BackendMetaData defineS3Backend()
{
return (new S3BackendMetaData()
.withBucketName(BaseS3Test.BUCKET_NAME)
.withBasePath(BaseS3Test.TEST_FOLDER)
.withName(BACKEND_NAME_S3));
}
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineS3CSVPersonTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_PERSON_S3)
.withLabel("Person S3 Table")
.withBackendName(defineS3Backend().getName())
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"))
.withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"))
.withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"))
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
.withField(new QFieldMetaData("email", QFieldType.STRING))
.withBackendDetails(new S3TableBackendDetails()
.withRecordFormat(RecordFormat.CSV)
.withCardinality(Cardinality.MANY)
);
}
/*******************************************************************************
**
*******************************************************************************/
public static QSession getMockSession()
{
MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule();
return (mockAuthenticationModule.createSession(null));
}
}

175
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java Normal file
View File

@ -0,0 +1,175 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local;
import java.io.File;
import java.io.IOException;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for FilesystemBackendModule
*******************************************************************************/
public class FilesystemBackendModuleTest
{
private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist";
@BeforeEach
public void beforeEach() throws IOException
{
new FilesystemActionTest().primeFilesystem();
}
@AfterEach
public void afterEach() throws Exception
{
new FilesystemActionTest().cleanFilesystem();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDeleteFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
/////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then delete one, then re-list, and assert that we have one fewer //
/////////////////////////////////////////////////////////////////////////////////////////////
List<File> filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
filesystemBackendModule.getActionBase().deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath());
List<File> filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeDelete.size() - 1, filesAfterDelete.size(),
"Should be one fewer file listed after deleting one.");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDeleteFileDoesNotExist() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
// note, our contract is that as long as the file doesn't exist after calling delete (e.g., if it wasn't there //
// to begin with, then we're okay, and don't expect an exception //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<File> filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
filesystemBackendModule.getActionBase().deleteFile(qInstance, table, PATH_THAT_WONT_EXIST);
List<File> filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeDelete.size(), filesAfterDelete.size(),
"Should be same number of files after deleting bogus path");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMoveFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
String subPath = basePath + File.separator + "subdir";
////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files (non-recursively) - then move one into a sub-folder, then re-list, and //
// assert that we have one fewer then list again including sub-folders, and see the changed count //
////////////////////////////////////////////////////////////////////////////////////////////////////
List<File> filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
String originalFilePath = filesBeforeMove.get(0).getAbsolutePath();
String movedFilePath = originalFilePath.replace(basePath, subPath);
filesystemBackendModule.getActionBase().moveFile(qInstance, table, originalFilePath, movedFilePath);
List<File> filesAfterMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeMove.size() - 1, filesAfterMove.size(), "Should be one fewer file in the listing after moving one.");
//////////////////////////////////////////////////////////////////////////
// move the file back and assert that the count goes back to the before //
//////////////////////////////////////////////////////////////////////////
filesystemBackendModule.getActionBase().moveFile(qInstance, table, movedFilePath, originalFilePath);
List<File> filesAfterMoveBack = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeMove.size(), filesAfterMoveBack.size(), "Should be original number of files after moving back");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMoveFileDoesNotExit() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
String subPath = basePath + File.separator + "subdir";
List<File> filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
String originalFilePath = filesBeforeMove.get(0).getAbsolutePath();
String movedFilePath = originalFilePath.replace(basePath, subPath);
Assertions.assertThrows(FilesystemException.class, () ->
{
FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
filesystemBackendModule.getActionBase().moveFile(qInstance, table, PATH_THAT_WONT_EXIST, movedFilePath);
});
}
}

128
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java Normal file
View File

@ -0,0 +1,128 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import static org.junit.jupiter.api.Assertions.fail;
/*******************************************************************************
** Base class for Filesystem action tests.
**
** Knows how to set up the filesystem for the tests.
*******************************************************************************/
public class FilesystemActionTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
public void beforeEach() throws Exception
{
primeFilesystem();
}
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
public void afterEach() throws Exception
{
cleanFilesystem();
}
/*******************************************************************************
** Set up the file system
*******************************************************************************/
public void primeFilesystem() throws IOException
{
TestUtils.cleanInstanceFiles();
TestUtils.increaseTestInstanceCounter();
FilesystemBackendMetaData filesystemBackendMetaData = TestUtils.defineLocalFilesystemBackend();
File baseDirectory = new File(filesystemBackendMetaData.getBasePath());
boolean mkdirsResult = baseDirectory.mkdirs();
if(!mkdirsResult)
{
fail("Failed to make directories at [" + baseDirectory + "] for filesystem backend module");
}
writePersonFiles(baseDirectory);
}
/*******************************************************************************
** Write some data files into the directory for the filesystem module.
*******************************************************************************/
private void writePersonFiles(File baseDirectory) throws IOException
{
String fullPath = baseDirectory.getAbsolutePath();
if (TestUtils.defineLocalFilesystemJSONPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
{
if (StringUtils.hasContent(details.getBasePath()))
{
fullPath += File.separatorChar + details.getBasePath();
}
}
fullPath += File.separatorChar;
String jsonData1 = """
[
{"id":1,"createDate":"2021-10-26 14:39:37","modifyDate":"2021-10-26 14:39:37","firstName":"John","lastName":"Doe","birthDate":"1981-01-01","email":"john@kingsrook.com"},
{"id":2,"createDate":"2022-06-17 14:52:59","modifyDate":"2022-06-17 14:52:59","firstName":"Jane","lastName":"Smith","birthDate":"1982-02-02","email":"jane@kingsrook.com"}
]
""";
FileUtils.writeStringToFile(new File(fullPath + "DATA-1.json"), jsonData1);
String jsonData2 = """
[
{"id":3,"createDate":"2021-11-27 15:40:38","modifyDate":"2021-11-27 15:40:38","firstName":"Homer","lastName":"S","birthDate":"1983-03-03","email":"homer.s@kingsrook.com"}
]
""";
FileUtils.writeStringToFile(new File(fullPath + "DATA-2.json"), jsonData2);
}
/*******************************************************************************
**
*******************************************************************************/
public void cleanFilesystem() throws IOException
{
TestUtils.cleanInstanceFiles();
}
}

68
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java Normal file
View File

@ -0,0 +1,68 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/*******************************************************************************
**
*******************************************************************************/
public class FilesystemQueryActionTest extends FilesystemActionTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQuery1() throws QException
{
QueryRequest queryRequest = initQueryRequest();
QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows");
Assertions.assertTrue(queryResult.getRecords().stream()
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
"All records should have a full-path in their backend details, matching the test folder name");
}
/*******************************************************************************
**
*******************************************************************************/
private QueryRequest initQueryRequest() throws QInstanceValidationException
{
QueryRequest queryRequest = new QueryRequest();
queryRequest.setInstance(TestUtils.defineInstance());
queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
return queryRequest;
}
}

77
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java Normal file
View File

@ -0,0 +1,77 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for FilesystemBackendMetaData
*******************************************************************************/
class FilesystemBackendMetaDataTest
{
/*******************************************************************************
** Test that an instance can be serialized as expected
*******************************************************************************/
@Test
public void testSerializingToJson() throws QInstanceValidationException
{
TestUtils.resetTestInstanceCounter();
QInstance qInstance = TestUtils.defineInstance();
String json = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance);
System.out.println(JsonUtils.prettyPrint(json));
System.out.println(json);
String expectToContain = """
"local-filesystem":{"basePath":"/tmp/filesystem-tests/0","backendType":"filesystem","name":"local-filesystem"}""";
assertTrue(json.contains(expectToContain));
}
/*******************************************************************************
** Test that an instance can be deserialized as expected
*******************************************************************************/
@Test
public void testDeserializingFromJson() throws IOException, QInstanceValidationException
{
QInstanceAdapter qInstanceAdapter = new QInstanceAdapter();
QInstance qInstance = TestUtils.defineInstance();
String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance);
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
assertThat(deserialized).usingRecursiveComparison()
.ignoringFields("hasBeenValidated") // note, this field is @JsonIgnore
.isEqualTo(qInstance);
}
}

228
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java Normal file
View File

@ -0,0 +1,228 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
import java.io.File;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for BasicETLCleanupSourceFilesFunction
*******************************************************************************/
public class BasicETLCleanupSourceFilesFunctionTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDelete1Record1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
testDelete(qInstance, List.of(filePath));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDelete2Records1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
testDelete(qInstance, List.of(filePath, filePath));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDelete2Record2File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath1 = getRandomFilePathPersonTable(qInstance);
String filePath2 = getRandomFilePathPersonTable(qInstance);
testDelete(qInstance, List.of(filePath1, filePath2));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMove1Record1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
testMove(qInstance, List.of(filePath));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMove2Records1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
testMove(qInstance, List.of(filePath, filePath));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMove2Record2File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath1 = getRandomFilePathPersonTable(qInstance);
String filePath2 = getRandomFilePathPersonTable(qInstance);
testMove(qInstance, List.of(filePath1, filePath2));
}
/*******************************************************************************
**
*******************************************************************************/
private void testDelete(QInstance qInstance, List<String> filePaths) throws Exception
{
RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_DELETE,
// todo - even though this field isn't needed, since we gave a value of "delete"
// the RunFunctionAction considers any missing input to be an error...
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, ""));
assertNull(runFunctionResult.getError());
for(String filePath : filePaths)
{
assertFalse(new File(filePath).exists(), "File should have been deleted.");
}
}
/*******************************************************************************
**
*******************************************************************************/
private void testMove(QInstance qInstance, List<String> filePaths) throws Exception
{
String trashDir = File.separator + "tmp" + File.separator + "trash";
RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_MOVE,
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir));
assertNull(runFunctionResult.getError());
for(String filePath : filePaths)
{
assertFalse(new File(filePath).exists(), "File should have been moved.");
String movedPath = trashDir + File.separator + (new File(filePath).getName());
assertTrue(new File(movedPath).exists(), "File should have been moved.");
}
}
/*******************************************************************************
**
*******************************************************************************/
private RunFunctionResult runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception
{
QFunctionMetaData qFunctionMetaData = new BasicETLCleanupSourceFilesFunction().defineFunctionMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addFunction(qFunctionMetaData);
qInstance.addProcess(qProcessMetaData);
HashSet<String> filePathsSet = new HashSet<>(filePaths);
for(String filePath : filePathsSet)
{
File file = new File(filePath);
FileUtils.writeStringToFile(file, "content");
}
// List<QRecord> records = filePaths.stream()
// .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList();
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(qFunctionMetaData.getName());
runFunctionRequest.setProcessName(qProcessMetaData.getName());
// runFunctionRequest.setRecords(records);
runFunctionRequest.setSession(TestUtils.getMockSession());
runFunctionRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
runFunctionRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
runFunctionRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
for(Map.Entry<String, String> entry : values.entrySet())
{
runFunctionRequest.addValue(entry.getKey(), entry.getValue());
}
RunFunctionAction runFunctionAction = new RunFunctionAction();
return (runFunctionAction.execute(runFunctionRequest));
}
/*******************************************************************************
**
*******************************************************************************/
private String getRandomFilePathPersonTable(QInstance qInstance)
{
FilesystemBackendMetaData backend = (FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS).getBackendDetails();
String tablePath = backend.getBasePath() + File.separator + backendDetails.getBasePath();
String filePath = tablePath + File.separator + UUID.randomUUID();
return filePath;
}
}

266
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java Normal file
View File

@ -0,0 +1,266 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync;
import java.util.List;
import java.util.stream.Collectors;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModuleSubclassForTest;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for FilesystemSyncProcess using S3 backend
*******************************************************************************/
class FilesystemSyncProcessS3Test extends BaseS3Test
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void test() throws Exception
{
QBackendModuleDispatcher.registerBackendModule(new S3BackendModuleSubclassForTest());
QInstance qInstance = TestUtils.defineInstance();
String sourceBucket = "source-bucket";
String archiveBucket = "archive-bucket";
String processingBucket = "processing-bucket";
getAmazonS3().createBucket(sourceBucket);
getAmazonS3().createBucket(archiveBucket);
getAmazonS3().createBucket(processingBucket);
S3BackendMetaData sourceBackend = defineBackend(qInstance, "source", sourceBucket);
S3BackendMetaData archiveBackend = defineBackend(qInstance, "archive", archiveBucket);
S3BackendMetaData processingBackend = defineBackend(qInstance, "processing", processingBucket);
QTableMetaData sourceTable = defineTable(qInstance, "source", sourceBackend, "source", "*/l3/*.csv");
QTableMetaData archiveTable = defineTable(qInstance, "archive", archiveBackend, "archive", "*/l3/*.csv");
QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
qInstance.addProcess(process);
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
///////////////////////////
// write some test files //
///////////////////////////
writeTestFile(sourceBackend, sourceTable, "foo/l3/1.csv", "x");
writeTestFile(sourceBackend, sourceTable, "bar/l3/2.csv", "x");
writeTestFile(archiveBackend, archiveTable, "foo/l3/1.csv", "x");
printTableListing(sourceBackend, sourceTable);
printTableListing(archiveBackend, archiveTable);
printTableListing(processingBackend, processingTable);
//////////////////////
// run the function //
//////////////////////
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(function.getName());
runFunctionRequest.setProcessName(process.getName());
runFunctionRequest.setSession(TestUtils.getMockSession());
RunFunctionAction runFunctionAction = new RunFunctionAction();
RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
System.out.println(runFunctionResult);
printTableListing(sourceBackend, sourceTable);
printTableListing(archiveBackend, archiveTable);
printTableListing(processingBackend, processingTable);
assertTableListing(archiveBackend, archiveTable, "root/archive/foo/l3/1.csv", "root/archive/bar/l3/2.csv");
assertTableListing(processingBackend, processingTable, "root/processing/bar/l3/2.csv");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testArchiveAndProcessingInSameBucket() throws Exception
{
QBackendModuleDispatcher.registerBackendModule(new S3BackendModuleSubclassForTest());
QInstance qInstance = TestUtils.defineInstance();
String vendorBucket = "vendor-bucket";
String localBucket = "local-bucket";
getAmazonS3().createBucket(vendorBucket);
getAmazonS3().createBucket(localBucket);
S3BackendMetaData vendorBackend = defineBackend(qInstance, "source", vendorBucket);
S3BackendMetaData localBackend = defineBackend(qInstance, "archive", localBucket);
QTableMetaData sourceTable = defineTable(qInstance, "source", vendorBackend, "source", "*/l3/*.csv");
QTableMetaData archiveTable = defineTable(qInstance, "archive", localBackend, "archive", "*/l3/*.csv");
QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
qInstance.addProcess(process);
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
///////////////////////////
// write some test files //
///////////////////////////
writeTestFile(vendorBackend, sourceTable, "foo/l3/1.csv", "x");
writeTestFile(vendorBackend, sourceTable, "bar/l3/2.csv", "x");
writeTestFile(localBackend, archiveTable, "foo/l3/1.csv", "x");
printTableListing(vendorBackend, sourceTable);
printTableListing(localBackend, archiveTable);
printTableListing(localBackend, processingTable);
//////////////////////
// run the function //
//////////////////////
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(function.getName());
runFunctionRequest.setProcessName(process.getName());
runFunctionRequest.setSession(TestUtils.getMockSession());
RunFunctionAction runFunctionAction = new RunFunctionAction();
RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
System.out.println(runFunctionResult);
printTableListing(vendorBackend, sourceTable);
printTableListing(localBackend, archiveTable);
printTableListing(localBackend, processingTable);
assertTableListing(localBackend, archiveTable, "root/archive/foo/l3/1.csv", "root/archive/bar/l3/2.csv");
assertTableListing(localBackend, processingTable, "root/processing/bar/l3/2.csv");
}
/*******************************************************************************
**
*******************************************************************************/
private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException
{
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
List<S3ObjectSummary> s3ObjectSummaries = actionBase.listFiles(table, backend);
assertEquals(paths.length, s3ObjectSummaries.size(), "Expected number of files in table: " + table.getName());
for(String path : paths)
{
assertTrue(s3ObjectSummaries.stream().anyMatch(s3o -> s3o.getKey().equals(path)),
"Path [" + path + "] should be in the listing, but was not. Full listing is: " +
s3ObjectSummaries.stream().map(S3ObjectSummary::getKey).collect(Collectors.joining(",")));
}
}
/*******************************************************************************
**
*******************************************************************************/
private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException
{
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
System.out.println("Files in: " + table.getName());
actionBase.listFiles(table, backend).forEach(o -> System.out.println(o.getKey()));
}
/*******************************************************************************
**
*******************************************************************************/
private void writeTestFile(S3BackendMetaData backend, QTableMetaData table, String name, String content) throws Exception
{
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
String fullPath = actionBase.getFullBasePath(table, backend);
actionBase.writeFile(backend, fullPath + "/" + name, content.getBytes());
}
/*******************************************************************************
**
*******************************************************************************/
private S3BackendMetaData defineBackend(QInstance qInstance, String which, String bucketName)
{
QBackendMetaData backendMetaData = new S3BackendMetaData()
.withBucketName(bucketName)
.withBasePath("root")
.withName("backend-" + which);
qInstance.addBackend(backendMetaData);
return (S3BackendMetaData) backendMetaData;
}
/*******************************************************************************
**
*******************************************************************************/
private QTableMetaData defineTable(QInstance qInstance, String which, QBackendMetaData backend, String path, String glob)
{
QTableMetaData qTableMetaData = new QTableMetaData()
.withName("table-" + which)
.withBackendName(backend.getName())
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withBackendDetails(new S3TableBackendDetails()
.withBasePath(path)
.withGlob(glob));
qInstance.addTable(qTableMetaData);
return (qTableMetaData);
}
}

123
src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java Normal file
View File

@ -0,0 +1,123 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync;
import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for FilesystemSyncProcess
*******************************************************************************/
class FilesystemSyncProcessTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void test() throws Exception
{
TestUtils.cleanInstanceFiles();
QTableMetaData sourceTable = defineTable("source");
QTableMetaData archiveTable = defineTable("archive");
QTableMetaData processingTable = defineTable("processing");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME);
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
// function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1);
QInstance qInstance = TestUtils.defineInstance();
qInstance.addTable(sourceTable);
qInstance.addTable(archiveTable);
qInstance.addTable(processingTable);
qInstance.addProcess(process);
///////////////////////////
// write some test files //
///////////////////////////
String basePath = ((FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS)).getBasePath();
writeTestFile(basePath, sourceTable, "1.txt", "x");
writeTestFile(basePath, sourceTable, "2.txt", "x");
// writeTestFile(basePath, sourceTable, "3.txt", "x");
writeTestFile(basePath, archiveTable, "2.txt", "x");
//////////////////////
// run the function //
//////////////////////
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(function.getName());
runFunctionRequest.setProcessName(process.getName());
runFunctionRequest.setSession(TestUtils.getMockSession());
RunFunctionAction runFunctionAction = new RunFunctionAction();
RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest);
System.out.println(runFunctionResult);
}
/*******************************************************************************
**
*******************************************************************************/
private void writeTestFile(String basePath, QTableMetaData table, String name, String content) throws IOException
{
String path = ((FilesystemTableBackendDetails) table.getBackendDetails()).getBasePath();
File file = new File(basePath + "/" + path + "/" + name);
FileUtils.writeStringToFile(file, content);
}
/*******************************************************************************
**
*******************************************************************************/
private QTableMetaData defineTable(String subPath)
{
return new QTableMetaData()
.withName("table-" + subPath)
.withBackendName(TestUtils.BACKEND_NAME_LOCAL_FS)
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath(subPath));
}
}

162
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java Normal file
View File

@ -0,0 +1,162 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3;
import cloud.localstack.ServiceName;
import cloud.localstack.awssdkv1.TestUtils;
import cloud.localstack.docker.LocalstackDockerExtension;
import cloud.localstack.docker.annotation.LocalstackDockerProperties;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.ExtendWith;
/*******************************************************************************
** Base class for tests that want to be able to work with localstack s3.
*******************************************************************************/
@ExtendWith(LocalstackDockerExtension.class)
@LocalstackDockerProperties(services = { ServiceName.S3 }, portEdge = "2960", portElasticSearch = "2961")
public class BaseS3Test
{
public static final String BUCKET_NAME = "localstack-test-bucket";
public static final String TEST_FOLDER = "test-files";
public static final String SUB_FOLDER = "sub-folder";
/*******************************************************************************
** Before each unit test, get the test bucket into a known state
*******************************************************************************/
@BeforeEach
public void beforeEach()
{
AmazonS3 amazonS3 = getAmazonS3();
amazonS3.createBucket(BUCKET_NAME);
amazonS3.putObject(BUCKET_NAME, "0.csv", getCSVHeader());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/1.csv", getCSVData1());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
}
/*******************************************************************************
** After each unit test, clean up the bucket
*******************************************************************************/
@AfterEach
public void afterEach()
{
AmazonS3 amazonS3 = getAmazonS3();
if(amazonS3.doesBucketExistV2(BUCKET_NAME))
{
////////////////////////
// todo - paginate... //
////////////////////////
for(S3ObjectSummary objectSummary : amazonS3.listObjectsV2(BUCKET_NAME).getObjectSummaries())
{
amazonS3.deleteObject(BUCKET_NAME, objectSummary.getKey());
}
amazonS3.deleteBucket(BUCKET_NAME);
}
}
/*******************************************************************************
** Access a localstack-configured s3 client.
*******************************************************************************/
protected AmazonS3 getAmazonS3()
{
return (TestUtils.getClientS3());
}
/*******************************************************************************
** Access the S3Utils object, with localstack-configured s3 client.
*******************************************************************************/
protected S3Utils getS3Utils()
{
S3Utils s3Utils = new S3Utils();
s3Utils.setAmazonS3(getAmazonS3());
return (s3Utils);
}
/*******************************************************************************
** Access a string of CSV test data.
*******************************************************************************/
protected String getCSVHeader()
{
return ("""
"id","createDate","modifyDate","firstName","lastName","birthDate","email"
""");
}
/*******************************************************************************
** Access a string of CSV test data.
*******************************************************************************/
protected String getCSVData1()
{
return (getCSVHeader() + """
"1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com"
"2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com"
""");
}
/*******************************************************************************
** Access a string of CSV test data.
*******************************************************************************/
protected String getCSVData2()
{
return (getCSVHeader() + """
"3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com"
"4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com"
"5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com"
""");
}
/*******************************************************************************
** Access a string of CSV test data.
*******************************************************************************/
protected String getCSVData3()
{
return (getCSVHeader() + """
"6","2022-06-20 15:31:02","2022-06-20 15:31:02","Lisa","S","1986-06-06","lisa.s@kingsrook.com"
""");
}
}

53
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleSubclassForTest.java Normal file
View File

@ -0,0 +1,53 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import cloud.localstack.awssdkv1.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
/*******************************************************************************
** Subclass of the S3Backend module, meant for use in unit tests, if/where we
** need to make sure we use the localstack version of the S3 client.
*******************************************************************************/
public class S3BackendModuleSubclassForTest extends S3BackendModule
{
/*******************************************************************************
** Seed the AbstractS3Action with an s3Utils object that has the localstack
** s3 client in it
*******************************************************************************/
@Override
public AbstractBaseFilesystemAction<S3ObjectSummary> getActionBase()
{
AbstractS3Action actionBase = (AbstractS3Action) super.getActionBase();
S3Utils s3Utils = new S3Utils();
s3Utils.setAmazonS3(TestUtils.getClientS3());
actionBase.setS3Utils(s3Utils);
return (actionBase);
}
}

156
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java Normal file
View File

@ -0,0 +1,156 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3;
import java.util.List;
import java.util.UUID;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for S3BackendModule
*******************************************************************************/
public class S3BackendModuleTest extends BaseS3Test
{
private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist";
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDeleteFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
/////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then delete one, then re-list, and assert that we have one fewer //
/////////////////////////////////////////////////////////////////////////////////////////////
List<S3ObjectSummary> s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
S3BackendModule s3BackendModule = new S3BackendModule();
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
actionBase.setS3Utils(getS3Utils());
actionBase.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey());
List<S3ObjectSummary> s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(),
"Should be one fewer file listed after deleting one.");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDeleteFileDoesNotExist() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
// note, we'd like to detect the non-delete, but there's no such info back from aws it appears? //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<S3ObjectSummary> s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
S3BackendModule s3BackendModule = new S3BackendModule();
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
actionBase.setS3Utils(getS3Utils());
actionBase.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST);
List<S3ObjectSummary> s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size(), s3ObjectSummariesAfterDelete.size(),
"Should be same number of files after deleting bogus path");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMoveFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
String subPath = TEST_FOLDER + "/" + SUB_FOLDER;
////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files (non-recursively) - then move one into a sub-folder, then re-list, and //
// assert that we have one fewer then list again including sub-folders, and see the changed count //
////////////////////////////////////////////////////////////////////////////////////////////////////
List<S3ObjectSummary> s3ObjectSummariesBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
List<S3ObjectSummary> s3ObjectSummariesInSubFolderBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, subPath, "");
List<S3ObjectSummary> s3ObjectSummariesRecursiveBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**");
S3BackendModule s3BackendModule = new S3BackendModule();
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
actionBase.setS3Utils(getS3Utils());
String key = s3ObjectSummariesBeforeMove.get(0).getKey();
actionBase.moveFile(qInstance, table, key, key.replaceFirst(TEST_FOLDER, subPath));
List<S3ObjectSummary> s3ObjectSummariesAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
List<S3ObjectSummary> s3ObjectSummariesRecursiveAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**");
List<S3ObjectSummary> s3ObjectSummariesInSubFolderAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, subPath, "");
Assertions.assertEquals(s3ObjectSummariesBeforeMove.size() - 1, s3ObjectSummariesAfterMove.size(),
"Should be one fewer file in the non-recursive listing after moving one.");
Assertions.assertEquals(s3ObjectSummariesRecursiveBeforeMove.size(), s3ObjectSummariesRecursiveAfterMove.size(),
"Should be same number of files in the recursive listing before and after the move");
Assertions.assertEquals(s3ObjectSummariesInSubFolderBeforeMove.size() + 1, s3ObjectSummariesInSubFolderAfterMove.size(),
"Should be one move file in the sub-folder listing after moving one.");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMoveFileDoesNotExit() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
String subPath = TEST_FOLDER + "/" + SUB_FOLDER;
S3BackendModule s3BackendModule = new S3BackendModule();
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
actionBase.setS3Utils(getS3Utils());
Assertions.assertThrows(FilesystemException.class, () ->
actionBase.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, subPath + "/" + UUID.randomUUID())
);
}
}

71
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java Normal file
View File

@ -0,0 +1,71 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/*******************************************************************************
**
*******************************************************************************/
public class S3QueryActionTest extends BaseS3Test
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQuery1() throws QException
{
QueryRequest queryRequest = initQueryRequest();
S3QueryAction s3QueryAction = new S3QueryAction();
s3QueryAction.setS3Utils(getS3Utils());
QueryResult queryResult = s3QueryAction.execute(queryRequest);
Assertions.assertEquals(5, queryResult.getRecords().size(), "Expected # of rows from unfiltered query");
Assertions.assertTrue(queryResult.getRecords().stream()
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseS3Test.TEST_FOLDER)),
"All records should have a full-path in their backend details, matching the test folder name");
}
/*******************************************************************************
**
*******************************************************************************/
private QueryRequest initQueryRequest() throws QInstanceValidationException
{
QueryRequest queryRequest = new QueryRequest();
queryRequest.setInstance(TestUtils.defineInstance());
queryRequest.setTableName(TestUtils.defineS3CSVPersonTable().getName());
return queryRequest;
}
}

77
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java Normal file
View File

@ -0,0 +1,77 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for S3BackendMetaData
*******************************************************************************/
class S3BackendMetaDataTest
{
/*******************************************************************************
** Test that an instance can be serialized as expected
*******************************************************************************/
@Test
public void testSerializingToJson() throws QInstanceValidationException
{
TestUtils.resetTestInstanceCounter();
QInstance qInstance = TestUtils.defineInstance();
String json = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance);
System.out.println(JsonUtils.prettyPrint(json));
System.out.println(json);
String expectToContain = """
{"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","secretKey":null,"accessKey":null,"backendType":"s3","name":"s3","region":null}""";
assertTrue(json.contains(expectToContain));
}
/*******************************************************************************
** Test that an instance can be deserialized as expected
*******************************************************************************/
@Test
public void testDeserializingFromJson() throws IOException, QInstanceValidationException
{
QInstanceAdapter qInstanceAdapter = new QInstanceAdapter();
QInstance qInstance = TestUtils.defineInstance();
String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance);
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
assertThat(deserialized).usingRecursiveComparison()
.ignoringFields("hasBeenValidated") // note, this field is @JsonIgnore
.isEqualTo(qInstance);
}
}

73
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/PathMatcherGlobTest.java Normal file
View File

@ -0,0 +1,73 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
import java.net.URI;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Verification for some of the behavior in the S3Utils - working with PathMatcher
** globs.
*******************************************************************************/
public class PathMatcherGlobTest
{
@Test
public void testPathMatcher() throws Exception
{
/////////////////////////////////////////////////////////////
// note: must start with for both the pattern and the uri //
/////////////////////////////////////////////////////////////
PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/*/acme/*/*.csv");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/1234.csv"))), "Glob should match");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/nj/acme/20220627/1234.csv"))), "Glob should match");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220628/1234.csv"))), "Glob should match");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/12345.csv"))), "Glob should match");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/beta/20220627/1234.csv"))), "Glob should not match (beta vs acme)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///something/stl/acme/20220627/1234.csv"))), "Glob should not match (wrong start path)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/csv"))), "Glob should not match (no file basename)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/1234.CSV"))), "Glob should not match (wrong case extension)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/extra/1234.csv"))), "Glob should not match (extra dir)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/extra/acme/20220627/1234.csv"))), "Glob should not match (extra dir)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/extra/stl/acme/20220627/1234.csv"))), "Glob should not match (extra dir)");
pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/**/acme/*/*.csv");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/extra/stl/acme/20220627/1234.csv"))), "Glob should match with extra dir");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/extra/extra2/stl/acme/20220627/1234.csv"))), "Glob should match with 2 extra dirs");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/acme/20220627/1234.csv"))), "Glob does not match with no dir for **");
pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/**");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/1234.csv"))), "Glob should match with extra dir");
pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/*");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///1234.csv"))), "Glob should match");
}
}

84
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java Normal file
View File

@ -0,0 +1,84 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
**
*******************************************************************************/
public class S3UtilsTest extends BaseS3Test
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testListObjectsInBucketAtPath()
{
S3Utils s3Utils = getS3Utils();
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders");
assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders");
assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders");
assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, "").size(), "Just in the subfolder non-recursive");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, "/**").size(), "Just in the subfolder recursive");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "//" + SUB_FOLDER, "/**").size(), "Just in the subfolder recursive, multi /");
assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "not-a-real-path/", "").size(), "In a non-existing folder");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string");
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testGetObjectAsInputStream() throws IOException
{
S3Utils s3Utils = getS3Utils();
List<S3ObjectSummary> s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", "");
S3ObjectSummary s3ObjectSummary = s3ObjectSummaries.stream().filter(o -> o.getKey().contains("1.csv")).findAny().get();
InputStream inputStream = s3Utils.getObjectAsInputStream(s3ObjectSummary);
String csvFromS3 = IOUtils.toString(inputStream);
assertEquals(getCSVData1(), csvFromS3, "File from S3 should match expected content");
}
}