From d60111466bf947ab8dc8bd668d33b27fa8ad5fdd Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 20 Jun 2022 16:07:50 -0500 Subject: [PATCH 01/30] QQQ-14 initial checkin --- .../actions/AbstractBaseFilesystemAction.java | 163 +++++++++++++++ .../AbstractFilesystemBackendMetaData.java | 80 ++++++++ ...AbstractFilesystemTableBackendDetails.java | 142 ++++++++++++++ .../local/FilesystemBackendModule.java | 119 +++++++++++ .../actions/AbstractFilesystemAction.java | 64 ++++++ .../local/actions/FilesystemDeleteAction.java | 59 ++++++ .../local/actions/FilesystemInsertAction.java | 65 ++++++ .../local/actions/FilesystemQueryAction.java | 45 +++++ .../local/actions/FilesystemUpdateAction.java | 65 ++++++ .../metadata/FilesystemBackendMetaData.java | 45 +++++ .../FilesystemTableBackendDetails.java | 44 +++++ .../module/filesystem/s3/S3BackendModule.java | 121 ++++++++++++ .../s3/actions/AbstractS3Action.java | 99 ++++++++++ .../filesystem/s3/actions/S3DeleteAction.java | 59 ++++++ .../filesystem/s3/actions/S3InsertAction.java | 65 ++++++ .../filesystem/s3/actions/S3QueryAction.java | 45 +++++ .../filesystem/s3/actions/S3UpdateAction.java | 65 ++++++ .../s3/model/metadata/S3BackendMetaData.java | 82 ++++++++ .../model/metadata/S3TableBackendDetails.java | 46 +++++ .../module/filesystem/s3/utils/S3Utils.java | 147 ++++++++++++++ .../backend/module/filesystem/TestUtils.java | 185 ++++++++++++++++++ .../local/actions/FilesystemActionTest.java | 103 ++++++++++ .../actions/FilesystemQueryActionTest.java | 88 +++++++++ .../FilesystemBackendMetaDataTest.java | 77 ++++++++ .../module/filesystem/s3/BaseS3Test.java | 173 ++++++++++++++++ .../s3/actions/S3QueryActionTest.java | 67 +++++++ .../model/metadata/S3BackendMetaDataTest.java | 77 ++++++++ .../filesystem/s3/utils/S3UtilsTest.java | 75 +++++++ 28 files changed, 2465 insertions(+) create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryAction.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemTableBackendDetails.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryAction.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3TableBackendDetails.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java create mode 100644 src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java create mode 100644 src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java create mode 100644 src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java create mode 100644 src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java create mode 100644 src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java create mode 100644 src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java create mode 100644 src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java create mode 100644 src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java new file mode 100644 index 00000000..07b5f3dc --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -0,0 +1,163 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.actions; + + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; +import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** Base class for all Filesystem actions across all modules. + *******************************************************************************/ +public abstract class AbstractBaseFilesystemAction +{ + + /******************************************************************************* + ** List the files for a table - to be implemented in module-specific subclasses. + *******************************************************************************/ + public abstract List listFiles(QTableMetaData table, QBackendMetaData backendBase); + + /******************************************************************************* + ** Read the contents of a file - to be implemented in module-specific subclasses. + *******************************************************************************/ + public abstract InputStream readFile(FILE file) throws IOException; + + + + /******************************************************************************* + ** Append together the backend's base path (if present), with a table's path (again, if present). + *******************************************************************************/ + protected String getFullPath(QTableMetaData table, QBackendMetaData backendBase) + { + AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase); + String fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : ""; + + AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); + if(StringUtils.hasContent(tableDetails.getPath())) + { + fullPath += File.separatorChar + tableDetails.getPath(); + } + + fullPath += File.separatorChar; + return fullPath; + } + + + + /******************************************************************************* + ** Get the backend metaData, type-checked as the requested type. + *******************************************************************************/ + protected T getBackendMetaData(Class outputClass, QBackendMetaData metaData) + { + if(!(outputClass.isInstance(metaData))) + { + throw new IllegalArgumentException("MetaData was not of expected type (was " + metaData.getClass().getSimpleName() + ")"); + } + return outputClass.cast(metaData); + } + + + + /******************************************************************************* + ** Get the backendDetails out of a table, type-checked as the requested type + *******************************************************************************/ + protected T getTableBackendDetails(Class outputClass, QTableMetaData tableMetaData) + { + QTableBackendDetails tableBackendDetails = tableMetaData.getBackendDetails(); + if(!(outputClass.isInstance(tableBackendDetails))) + { + throw new IllegalArgumentException("Table backend details was not of expected type (was " + tableBackendDetails.getClass().getSimpleName() + ")"); + } + return outputClass.cast(tableBackendDetails); + } + + + + /******************************************************************************* + ** Generic implementation of the execute method from the QueryInterface + *******************************************************************************/ + public QueryResult executeQuery(QueryRequest queryRequest) throws QException + { + try + { + QueryResult rs = new QueryResult(); + List records = new ArrayList<>(); + rs.setRecords(records); + + QTableMetaData table = queryRequest.getTable(); + AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); + List files = listFiles(table, queryRequest.getBackend()); + + for(FILE file : files) + { + switch(tableDetails.getRecordFormat()) + { + case "csv": + { + String fileContents = IOUtils.toString(readFile(file)); + List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); + + records.addAll(recordsInFile); + break; + } + case "json": + { + String fileContents = IOUtils.toString(readFile(file)); + List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); + + records.addAll(recordsInFile); + break; + } + default: + { + throw new NotImplementedException("Filesystem record format " + tableDetails.getRecordFormat() + " is not yet implemented"); + } + } + } + + return rs; + } + catch(Exception e) + { + e.printStackTrace(); + throw new QException("Error executing query", e); + } + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java new file mode 100644 index 00000000..9002bdb8 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java @@ -0,0 +1,80 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; + + +/******************************************************************************* + ** Base class for all BackendMetaData for all filesystem-style backend modules. + *******************************************************************************/ +public class AbstractFilesystemBackendMetaData extends QBackendMetaData +{ + private String basePath; + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public AbstractFilesystemBackendMetaData() + { + super(); + } + + + + /******************************************************************************* + ** Getter for basePath + ** + *******************************************************************************/ + public String getBasePath() + { + return (basePath); + } + + + + /******************************************************************************* + ** Setter for basePath + ** + *******************************************************************************/ + public void setBasePath(String basePath) + { + this.basePath = basePath; + } + + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withBasePath(String basePath) + { + this.basePath = basePath; + return (T) this; + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java new file mode 100644 index 00000000..ca5eae58 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java @@ -0,0 +1,142 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; + + +/******************************************************************************* + ** Extension of QTableBackendDetails, with details specific to a Filesystem table. + *******************************************************************************/ +public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails +{ + private String path; + private String recordFormat; // todo - enum? but hard w/ serialization? + private String cardinality; // todo - enum? + + + + /******************************************************************************* + ** Getter for path + ** + *******************************************************************************/ + public String getPath() + { + return path; + } + + + + /******************************************************************************* + ** Setter for path + ** + *******************************************************************************/ + public void setPath(String path) + { + this.path = path; + } + + + + /******************************************************************************* + ** Fluent Setter for path + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withPath(String path) + { + this.path = path; + return (T) this; + } + + + + /******************************************************************************* + ** Getter for recordFormat + ** + *******************************************************************************/ + public String getRecordFormat() + { + return recordFormat; + } + + + + /******************************************************************************* + ** Setter for recordFormat + ** + *******************************************************************************/ + public void setRecordFormat(String recordFormat) + { + this.recordFormat = recordFormat; + } + + + + /******************************************************************************* + ** Fluent Setter for recordFormat + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withRecordFormat(String recordFormat) + { + this.recordFormat = recordFormat; + return ((T) this); + } + + + + /******************************************************************************* + ** Getter for cardinality + ** + *******************************************************************************/ + public String getCardinality() + { + return cardinality; + } + + + + /******************************************************************************* + ** Setter for cardinality + ** + *******************************************************************************/ + public void setCardinality(String cardinality) + { + this.cardinality = cardinality; + } + + + + /******************************************************************************* + ** Fluent Setter for cardinality + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withCardinality(String cardinality) + { + this.cardinality = cardinality; + return ((T) this); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java new file mode 100644 index 00000000..25a9e8c7 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java @@ -0,0 +1,119 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package com.kingsrook.qqq.backend.module.filesystem.local; + + +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemUpdateAction; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; + + +/******************************************************************************* + ** QQQ Backend module for working with (local) Filesystems. + *******************************************************************************/ +public class FilesystemBackendModule implements QBackendModuleInterface +{ + + + /******************************************************************************* + ** Method where a backend module must be able to provide its type (name). + *******************************************************************************/ + @Override + public String getBackendType() + { + return ("filesystem"); + } + + + + /******************************************************************************* + ** Method to identify the class used for backend meta data for this module. + *******************************************************************************/ + @Override + public Class getBackendMetaDataClass() + { + return (FilesystemBackendMetaData.class); + } + + + /******************************************************************************* + ** Method to identify the class used for table-backend details for this module. + *******************************************************************************/ + @Override + public Class getTableBackendDetailsClass() + { + return FilesystemTableBackendDetails.class; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryInterface getQueryInterface() + { + return new FilesystemQueryAction(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InsertInterface getInsertInterface() + { + return (new FilesystemInsertAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public UpdateInterface getUpdateInterface() + { + return (new FilesystemUpdateAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public DeleteInterface getDeleteInterface() + { + return (new FilesystemDeleteAction()); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java new file mode 100644 index 00000000..fde5c1d5 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -0,0 +1,64 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; + + +/******************************************************************************* + ** Base class for all (local) Filesystem actions + *******************************************************************************/ +public class AbstractFilesystemAction extends AbstractBaseFilesystemAction +{ + + /******************************************************************************* + ** List the files for this table. + *******************************************************************************/ + @Override + public List listFiles(QTableMetaData table, QBackendMetaData backendBase) + { + String fullPath = getFullPath(table, backendBase); + File directory = new File(fullPath); + return Arrays.asList(directory.listFiles()); + } + + + + /******************************************************************************* + ** Read the contents of a file. + *******************************************************************************/ + @Override + public InputStream readFile(File file) throws IOException + { + return (new FileInputStream(file)); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java new file mode 100644 index 00000000..4d6d9f92 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java @@ -0,0 +1,59 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest; +import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilesystemDeleteAction implements DeleteInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public DeleteResult execute(DeleteRequest deleteRequest) throws QException + { + try + { + DeleteResult rs = new DeleteResult(); + QTableMetaData table = deleteRequest.getTable(); + + throw new NotImplementedException("Filesystem delete not implemented"); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing delete: " + e.getMessage(), e); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java new file mode 100644 index 00000000..328d7b43 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java @@ -0,0 +1,65 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest; +import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult; +import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilesystemInsertAction implements InsertInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public InsertResult execute(InsertRequest insertRequest) throws QException + { + try + { + InsertResult rs = new InsertResult(); + QTableMetaData table = insertRequest.getTable(); + + List recordsWithStatus = new ArrayList<>(); + rs.setRecords(recordsWithStatus); + + throw new NotImplementedException("Filesystem insert not implemented"); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing insert: " + e.getMessage(), e); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryAction.java new file mode 100644 index 00000000..56dc6d5d --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; +import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilesystemQueryAction extends AbstractFilesystemAction implements QueryInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public QueryResult execute(QueryRequest queryRequest) throws QException + { + return executeQuery(queryRequest); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java new file mode 100644 index 00000000..bae14d1b --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java @@ -0,0 +1,65 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest; +import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult; +import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilesystemUpdateAction implements UpdateInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public UpdateResult execute(UpdateRequest updateRequest) throws QException + { + try + { + UpdateResult rs = new UpdateResult(); + QTableMetaData table = updateRequest.getTable(); + + List recordsWithStatus = new ArrayList<>(); + rs.setRecords(recordsWithStatus); + + throw new NotImplementedException("Filesystem update not implemented"); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing update: " + e.getMessage(), e); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java new file mode 100644 index 00000000..02b3950e --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule; + + +/******************************************************************************* + ** (local) Filesystem backend meta data. + *******************************************************************************/ +public class FilesystemBackendMetaData extends AbstractFilesystemBackendMetaData +{ + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public FilesystemBackendMetaData() + { + super(); + setBackendType(FilesystemBackendModule.class); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemTableBackendDetails.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemTableBackendDetails.java new file mode 100644 index 00000000..b2f8cee9 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemTableBackendDetails.java @@ -0,0 +1,44 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule; + + +/******************************************************************************* + ** (local) Filesystem specific Extension of QTableBackendDetails + *******************************************************************************/ +public class FilesystemTableBackendDetails extends AbstractFilesystemTableBackendDetails +{ + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public FilesystemTableBackendDetails() + { + super(); + setBackendType(FilesystemBackendModule.class); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java new file mode 100644 index 00000000..7c574cf3 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java @@ -0,0 +1,121 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3; + + +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; + + +/******************************************************************************* + ** QQQ Backend module for working with AWS S3 filesystems + *******************************************************************************/ +public class S3BackendModule implements QBackendModuleInterface +{ + + + /******************************************************************************* + ** Method where a backend module must be able to provide its type (name). + *******************************************************************************/ + @Override + public String getBackendType() + { + return ("s3"); + } + + + + /******************************************************************************* + ** Method to identify the class used for backend meta data for this module. + *******************************************************************************/ + @Override + public Class getBackendMetaDataClass() + { + return (S3BackendMetaData.class); + } + + + + /******************************************************************************* + ** Method to identify the class used for table-backend details for this module. + *******************************************************************************/ + @Override + public Class getTableBackendDetailsClass() + { + return S3TableBackendDetails.class; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryInterface getQueryInterface() + { + return new S3QueryAction(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InsertInterface getInsertInterface() + { + return (new S3InsertAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public UpdateInterface getUpdateInterface() + { + return (new S3UpdateAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public DeleteInterface getDeleteInterface() + { + return (new S3DeleteAction()); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java new file mode 100644 index 00000000..12c69959 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -0,0 +1,99 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; + + +/******************************************************************************* + ** Base class for all S3 filesystem actions + *******************************************************************************/ +public class AbstractS3Action extends AbstractBaseFilesystemAction +{ + private S3Utils s3Utils; + + + + /******************************************************************************* + ** Set the S3Utils object. + *******************************************************************************/ + public void setS3Utils(S3Utils s3Utils) + { + this.s3Utils = s3Utils; + } + + + + /******************************************************************************* + ** Internal accessor for the s3Utils object - should always use this, not the field. + *******************************************************************************/ + private S3Utils getS3Utils() + { + if(s3Utils == null) + { + s3Utils = new S3Utils(); + } + + return s3Utils; + } + + + + /******************************************************************************* + ** List the files for a table. + *******************************************************************************/ + @Override + public List listFiles(QTableMetaData table, QBackendMetaData backendBase) + { + S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase); + + String fullPath = getFullPath(table, backendBase); + String bucketName = s3BackendMetaData.getBucketName(); + + //////////////////////////////////////////////////////////////////// + // todo - read metadata to decide if we should include subfolders // + // todo - look at metadata to configure the s3 client here? // + //////////////////////////////////////////////////////////////////// + boolean includeSubfolders = false; + return getS3Utils().listObjectsInBucketAtPath(bucketName, fullPath, includeSubfolders); + } + + + + /******************************************************************************* + ** Read the contents of a file. + *******************************************************************************/ + @Override + public InputStream readFile(S3ObjectSummary s3ObjectSummary) throws IOException + { + return (getS3Utils().getObjectAsInputStream(s3ObjectSummary)); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java new file mode 100644 index 00000000..8598c10e --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java @@ -0,0 +1,59 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest; +import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3DeleteAction implements DeleteInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public DeleteResult execute(DeleteRequest deleteRequest) throws QException + { + try + { + DeleteResult rs = new DeleteResult(); + QTableMetaData table = deleteRequest.getTable(); + + throw new NotImplementedException("S3 delete not implemented"); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing delete: " + e.getMessage(), e); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java new file mode 100644 index 00000000..310709b1 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java @@ -0,0 +1,65 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest; +import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult; +import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3InsertAction implements InsertInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public InsertResult execute(InsertRequest insertRequest) throws QException + { + try + { + InsertResult rs = new InsertResult(); + QTableMetaData table = insertRequest.getTable(); + + List recordsWithStatus = new ArrayList<>(); + rs.setRecords(recordsWithStatus); + + throw new NotImplementedException("S3 insert not implemented"); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing insert: " + e.getMessage(), e); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryAction.java new file mode 100644 index 00000000..46e71e25 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; +import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3QueryAction extends AbstractS3Action implements QueryInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public QueryResult execute(QueryRequest queryRequest) throws QException + { + return (super.executeQuery(queryRequest)); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java new file mode 100644 index 00000000..31844f47 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java @@ -0,0 +1,65 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest; +import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult; +import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3UpdateAction implements UpdateInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public UpdateResult execute(UpdateRequest updateRequest) throws QException + { + try + { + UpdateResult rs = new UpdateResult(); + QTableMetaData table = updateRequest.getTable(); + + List recordsWithStatus = new ArrayList<>(); + rs.setRecords(recordsWithStatus); + + throw new NotImplementedException("S3 update not implemented"); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing update: " + e.getMessage(), e); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java new file mode 100644 index 00000000..70bbe0ff --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java @@ -0,0 +1,82 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; + + +/******************************************************************************* + ** (local) Filesystem backend meta data. + *******************************************************************************/ +public class S3BackendMetaData extends AbstractFilesystemBackendMetaData +{ + private String bucketName; + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public S3BackendMetaData() + { + super(); + setBackendType(S3BackendModule.class); + } + + + + /******************************************************************************* + ** Getter for bucketName + ** + *******************************************************************************/ + public String getBucketName() + { + return bucketName; + } + + + + /******************************************************************************* + ** Setter for bucketName + ** + *******************************************************************************/ + public void setBucketName(String bucketName) + { + this.bucketName = bucketName; + } + + + + /******************************************************************************* + ** Fluent setter for bucketName + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withBucketName(String bucketName) + { + this.bucketName = bucketName; + return (T) this; + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3TableBackendDetails.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3TableBackendDetails.java new file mode 100644 index 00000000..ba372cc6 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3TableBackendDetails.java @@ -0,0 +1,46 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; + + +/******************************************************************************* + ** S3 specific Extension of QTableBackendDetails + *******************************************************************************/ +public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails +{ + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public S3TableBackendDetails() + { + super(); + setBackendType(S3BackendModule.class); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java new file mode 100644 index 00000000..4dfd4f1c --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java @@ -0,0 +1,147 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.utils; + + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Utility methods for working with AWS S3. + *******************************************************************************/ +public class S3Utils +{ + private static final Logger LOG = LogManager.getLogger(S3Utils.class); + + private AmazonS3 s3; + + + + /******************************************************************************* + ** List the objects in an S3 bucket at a given path + *******************************************************************************/ + public List listObjectsInBucketAtPath(String bucketName, String fullPath, boolean includeSubfolders) + { + ////////////////////////////////////////////////////////////////////////////////////////////////// + // s3 list requests find nothing if the path starts with a /, so strip away any leading slashes // + // also strip away trailing /'s, for consistent known paths. // + // also normalize any duplicated /'s to a single /. // + ////////////////////////////////////////////////////////////////////////////////////////////////// + fullPath = fullPath.replaceFirst("^/+", "").replaceFirst("/+$", "").replaceAll("//+", "/"); + + ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request() + .withBucketName(bucketName) + .withPrefix(fullPath); + + ListObjectsV2Result listObjectsV2Result = null; + List rs = new ArrayList<>(); + + do + { + if(listObjectsV2Result != null) + { + listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken()); + } + listObjectsV2Result = getS3().listObjectsV2(listObjectsV2Request); + + ////////////////////////////////// + // put files in the result list // + ////////////////////////////////// + for(S3ObjectSummary objectSummary : listObjectsV2Result.getObjectSummaries()) + { + String key = objectSummary.getKey(); + + ////////////////// + // skip folders // + ////////////////// + if(key.endsWith("/")) + { + LOG.debug("Skipping file [{}] because it is a folder", key); + continue; + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we're not supposed to include subfolders, check the path on this file, and only include it if it matches the request // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(!includeSubfolders) + { + String prefix = key.substring(0, key.lastIndexOf("/")); + if(!prefix.equals(fullPath)) + { + LOG.debug("Skipping file [{}] in a sub-folder [{}] != [{}]", key, prefix, fullPath); + continue; + } + } + + rs.add(objectSummary); + } + } + while(listObjectsV2Result.isTruncated()); + + return rs; + } + + + /******************************************************************************* + ** + *******************************************************************************/ + public InputStream getObjectAsInputStream(S3ObjectSummary s3ObjectSummary) + { + return getS3().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()).getObjectContent(); + } + + + /******************************************************************************* + ** Setter for AmazonS3 client object. + *******************************************************************************/ + public void setAmazonS3(AmazonS3 s3) + { + this.s3 = s3; + } + + + + /******************************************************************************* + ** Getter for AmazonS3 client object. + *******************************************************************************/ + public AmazonS3 getS3() + { + if(s3 == null) + { + s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build(); + } + + return s3; + } + + +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java new file mode 100644 index 00000000..c624b4a7 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -0,0 +1,185 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem; + + +import java.io.File; +import java.io.IOException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import org.apache.commons.io.FileUtils; + + +/******************************************************************************* + ** Utility methods for Filesystem module tests + *******************************************************************************/ +public class TestUtils +{ + private static int testInstanceCounter = 0; + private final static String BASE_PATH = "/tmp/filesystem-tests"; + + + + /******************************************************************************* + ** Meant to be called in a @BeforeEach - increment an internal counter that will + ** give us a unique directory name for each test method. + *******************************************************************************/ + public static void increaseTestInstanceCounter() + { + testInstanceCounter++; + } + + + + /******************************************************************************* + ** Reset the counter to 0 (e.g., to let some tests have a known value). + *******************************************************************************/ + public static void resetTestInstanceCounter() + { + testInstanceCounter = 0; + } + + + + /******************************************************************************* + ** Get the current value of the testInstanceCounter. See {@link #increaseTestInstanceCounter()} + *******************************************************************************/ + public static int getTestInstanceCounter() + { + return (testInstanceCounter); + } + + + + /******************************************************************************* + ** Meant to run both after and before test methods - makes sure the file system + ** is empty for the path under the instance. + *******************************************************************************/ + public static void cleanInstanceFiles() throws IOException + { + FileUtils.deleteDirectory(new File(BASE_PATH + File.separator + testInstanceCounter)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QInstance defineInstance() throws QInstanceValidationException + { + QInstance qInstance = new QInstance(); + qInstance.addBackend(defineLocalFilesystemBackend()); + qInstance.addTable(defineLocalFilesystemCSVPersonTable()); + qInstance.addBackend(defineS3Backend()); + qInstance.addTable(defineS3CSVPersonTable()); + + new QInstanceValidator().validate(qInstance); + + return (qInstance); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static FilesystemBackendMetaData defineLocalFilesystemBackend() + { + return (new FilesystemBackendMetaData() + .withBasePath(BASE_PATH + File.separator + testInstanceCounter) + .withName("local-filesystem")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineLocalFilesystemCSVPersonTable() + { + return new QTableMetaData() + .withName("person") + .withLabel("Person") + .withBackendName(defineLocalFilesystemBackend().getName()) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date")) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) + .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name")) + .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name")) + .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) + .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withBackendDetails(new FilesystemTableBackendDetails() + .withPath("persons") + .withRecordFormat("csv") + .withCardinality("many") + ); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static S3BackendMetaData defineS3Backend() + { + return (new S3BackendMetaData() + .withBucketName(BaseS3Test.BUCKET_NAME) + .withBasePath(BaseS3Test.TEST_FOLDER) + .withName("s3")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineS3CSVPersonTable() + { + return new QTableMetaData() + .withName("person-s3") + .withLabel("Person S3 Table") + .withBackendName(defineS3Backend().getName()) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date")) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) + .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name")) + .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name")) + .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) + .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withBackendDetails(new S3TableBackendDetails() + .withRecordFormat("csv") + .withCardinality("many") + ); + } + +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java new file mode 100644 index 00000000..7d19512e --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java @@ -0,0 +1,103 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import java.io.File; +import java.io.IOException; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import org.apache.commons.io.FileUtils; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** Base class for Filesystem action tests. + ** + ** Knows how to set up the filesystem for the tests. + *******************************************************************************/ +public class FilesystemActionTest +{ + + /******************************************************************************* + ** Set up the file system + *******************************************************************************/ + protected void primeFilesystem() throws IOException + { + TestUtils.cleanInstanceFiles(); + TestUtils.increaseTestInstanceCounter(); + FilesystemBackendMetaData filesystemBackendMetaData = TestUtils.defineLocalFilesystemBackend(); + + File baseDirectory = new File(filesystemBackendMetaData.getBasePath()); + boolean mkdirsResult = baseDirectory.mkdirs(); + if(!mkdirsResult) + { + fail("Failed to make directories at [" + baseDirectory + "] for filesystem backend module"); + } + + writePersonFiles(baseDirectory); + } + + + + private void writePersonFiles(File baseDirectory) throws IOException + { + String fullPath = baseDirectory.getAbsolutePath(); + if (TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details) + { + if (StringUtils.hasContent(details.getPath())) + { + fullPath += File.separatorChar + details.getPath(); + } + } + fullPath += File.separatorChar; + + String csvHeader = """ + "id","createDate","modifyDate","firstName","lastName","birthDate","email" + """; + + String csvData1 = csvHeader + """ + "1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com" + "2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com" + """; + FileUtils.writeStringToFile(new File(fullPath + "DATA-1.csv"), csvData1); + + String csvData2 = csvHeader + """ + "3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com" + "4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com" + "5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com" + """; + FileUtils.writeStringToFile(new File(fullPath + "DATA-2.csv"), csvData2); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void cleanFilesystem() throws IOException + { + TestUtils.cleanInstanceFiles(); + } +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java new file mode 100644 index 00000000..b918e457 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java @@ -0,0 +1,88 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilesystemQueryActionTest extends FilesystemActionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + public void beforeEach() throws Exception + { + super.primeFilesystem(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + public void afterEach() throws Exception + { + super.cleanFilesystem(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQuery1() throws QException + { + QueryRequest queryRequest = initQueryRequest(); + QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest); + Assertions.assertEquals(5, queryResult.getRecords().size(), "Unfiltered query should find all rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QueryRequest initQueryRequest() throws QInstanceValidationException + { + QueryRequest queryRequest = new QueryRequest(); + queryRequest.setInstance(TestUtils.defineInstance()); + queryRequest.setTableName(TestUtils.defineLocalFilesystemCSVPersonTable().getName()); + return queryRequest; + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java new file mode 100644 index 00000000..e8b24695 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata; + + +import java.io.IOException; +import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for FilesystemBackendMetaData + *******************************************************************************/ +class FilesystemBackendMetaDataTest +{ + + + /******************************************************************************* + ** Test that an instance can be serialized as expected + *******************************************************************************/ + @Test + public void testSerializingToJson() throws QInstanceValidationException + { + TestUtils.resetTestInstanceCounter(); + QInstance qInstance = TestUtils.defineInstance(); + String json = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance); + System.out.println(JsonUtils.prettyPrint(json)); + System.out.println(json); + String expectToContain = """ + "local-filesystem":{"basePath":"/tmp/filesystem-tests/0","backendType":"filesystem","name":"local-filesystem"}"""; + assertTrue(json.contains(expectToContain)); + } + + + + /******************************************************************************* + ** Test that an instance can be deserialized as expected + *******************************************************************************/ + @Test + public void testDeserializingFromJson() throws IOException, QInstanceValidationException + { + QInstanceAdapter qInstanceAdapter = new QInstanceAdapter(); + + QInstance qInstance = TestUtils.defineInstance(); + String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance); + + QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json); + assertThat(deserialized).usingRecursiveComparison() + .ignoringFields("hasBeenValidated") + .isEqualTo(qInstance); + } +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java new file mode 100644 index 00000000..3fb38d3a --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java @@ -0,0 +1,173 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3; + + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + + +/******************************************************************************* + ** + *******************************************************************************/ +@Testcontainers +public class BaseS3Test +{ + private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("localstack/localstack"); + + public static final String BUCKET_NAME = "localstack-test-bucket"; + public static final String TEST_FOLDER = "test-files"; + public static final String SUB_FOLDER = "sub-folder"; + + @Container + private static LocalStackContainer localStack = new LocalStackContainer(DEFAULT_IMAGE_NAME) + .withServices(LocalStackContainer.Service.S3); + + + + /******************************************************************************* + ** Before each unit test, get the test bucket into a known state + *******************************************************************************/ + @BeforeEach + public void beforeEach() + { + AmazonS3 amazonS3 = getAmazonS3(); + + amazonS3.createBucket(BUCKET_NAME); + amazonS3.putObject(BUCKET_NAME, "0.csv", getCSVHeader()); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/1.csv", getCSVData1()); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2()); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3()); + } + + + + /******************************************************************************* + ** After each unit test, clean up the bucket + *******************************************************************************/ + @AfterEach + public void afterEach() + { + AmazonS3 amazonS3 = getAmazonS3(); + + if(amazonS3.doesBucketExistV2(BUCKET_NAME)) + { + //////////////////////// + // todo - paginate... // + //////////////////////// + for(S3ObjectSummary objectSummary : amazonS3.listObjectsV2(BUCKET_NAME).getObjectSummaries()) + { + amazonS3.deleteObject(BUCKET_NAME, objectSummary.getKey()); + } + amazonS3.deleteBucket(BUCKET_NAME); + } + } + + + + /******************************************************************************* + ** Access a localstack-configured s3 client. + *******************************************************************************/ + protected AmazonS3 getAmazonS3() + { + BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(localStack.getAccessKey(), localStack.getSecretKey()); + AmazonS3 amazonS3 = AmazonS3ClientBuilder.standard() + .withEndpointConfiguration(localStack.getEndpointConfiguration(LocalStackContainer.Service.S3)) + .withCredentials(new AWSStaticCredentialsProvider(basicAWSCredentials)) + .build(); + return (amazonS3); + } + + + + /******************************************************************************* + ** Access the S3Utils object, with localstack-configured s3 client. + *******************************************************************************/ + protected S3Utils getS3Utils() + { + S3Utils s3Utils = new S3Utils(); + s3Utils.setAmazonS3(getAmazonS3()); + return (s3Utils); + } + + + + /******************************************************************************* + ** Access a string of CSV test data. + *******************************************************************************/ + protected String getCSVHeader() + { + return (""" + "id","createDate","modifyDate","firstName","lastName","birthDate","email" + """); + } + + + + /******************************************************************************* + ** Access a string of CSV test data. + *******************************************************************************/ + protected String getCSVData1() + { + return (getCSVHeader() + """ + "1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com" + "2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com" + """); + } + + + + /******************************************************************************* + ** Access a string of CSV test data. + *******************************************************************************/ + protected String getCSVData2() + { + return (getCSVHeader() + """ + "3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com" + "4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com" + "5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com" + """); + } + + + + /******************************************************************************* + ** Access a string of CSV test data. + *******************************************************************************/ + protected String getCSVData3() + { + return (getCSVHeader() + """ + "6","2022-06-20 15:31:02","2022-06-20 15:31:02","Lisa","S","1986-06-06","lisa.s@kingsrook.com" + """); + } + +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java new file mode 100644 index 00000000..6fe9d2e0 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3QueryActionTest extends BaseS3Test +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQuery1() throws QException + { + QueryRequest queryRequest = initQueryRequest(); + S3QueryAction s3QueryAction = new S3QueryAction(); + s3QueryAction.setS3Utils(getS3Utils()); + QueryResult queryResult = s3QueryAction.execute(queryRequest); + Assertions.assertEquals(5, queryResult.getRecords().size(), "Expected # of rows from unfiltered query"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QueryRequest initQueryRequest() throws QInstanceValidationException + { + QueryRequest queryRequest = new QueryRequest(); + queryRequest.setInstance(TestUtils.defineInstance()); + queryRequest.setTableName(TestUtils.defineS3CSVPersonTable().getName()); + return queryRequest; + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java new file mode 100644 index 00000000..6211d4ce --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata; + + +import java.io.IOException; +import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for S3BackendMetaData + *******************************************************************************/ +class S3BackendMetaDataTest +{ + + + /******************************************************************************* + ** Test that an instance can be serialized as expected + *******************************************************************************/ + @Test + public void testSerializingToJson() throws QInstanceValidationException + { + TestUtils.resetTestInstanceCounter(); + QInstance qInstance = TestUtils.defineInstance(); + String json = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance); + System.out.println(JsonUtils.prettyPrint(json)); + System.out.println(json); + String expectToContain = """ + {"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","backendType":"s3","name":"s3"}"""; + assertTrue(json.contains(expectToContain)); + } + + + + /******************************************************************************* + ** Test that an instance can be deserialized as expected + *******************************************************************************/ + @Test + public void testDeserializingFromJson() throws IOException, QInstanceValidationException + { + QInstanceAdapter qInstanceAdapter = new QInstanceAdapter(); + + QInstance qInstance = TestUtils.defineInstance(); + String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance); + + QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json); + assertThat(deserialized).usingRecursiveComparison() + .ignoringFields("hasBeenValidated") + .isEqualTo(qInstance); + } +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java new file mode 100644 index 00000000..0de6c5e2 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java @@ -0,0 +1,75 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.utils; + + +import java.io.IOException; +import java.io.InputStream; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3UtilsTest extends BaseS3Test +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testListObjectsInBucketAtPath() + { + S3Utils s3Utils = getS3Utils(); + assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/", false).size(), "Expected # of s3 objects without subfolders"); + assertEquals(3, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/", true).size(), "Expected # of s3 objects with subfolders"); + assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "/" + TEST_FOLDER + "/", false).size(), "With leading slash"); + assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "/" + TEST_FOLDER, false).size(), "Without trailing slash"); + assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "//" + TEST_FOLDER + "//", false).size(), "With multiple leading and trailing slashes"); + assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, false).size(), "Just in the subfolder non-recursive"); + assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, true).size(), "Just in the subfolder recursive"); + assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "//" + SUB_FOLDER, true).size(), "Just in the subfolder recursive"); + assertEquals(0, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "not-a-real-path/", true).size(), "In a non-existing folder"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testGetObjectAsInputStream() throws IOException + { + S3Utils s3Utils = getS3Utils(); + S3ObjectSummary s3ObjectSummary = s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "test-files", true).get(0); + InputStream inputStream = s3Utils.getObjectAsInputStream(s3ObjectSummary); + String csvFromS3 = IOUtils.toString(inputStream); + + // todo - should check the filename somewhere, right? + assertEquals(getCSVData1(), csvFromS3, "File from S3 should match expected content"); + } + +} \ No newline at end of file From 34b0fe2a4e6c164d9f233c080bdf5b197373a0cf Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 20 Jun 2022 16:08:24 -0500 Subject: [PATCH 02/30] QQQ-14 update qqq-backend-core snapshot --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 51e09ceb..2b3fc743 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ com.kingsrook.qqq qqq-backend-core - 0.0.0-SNAPSHOT + 0.0.0-20220620.140542-7 From 69699b410489a65c0a30e6f5355ff5f12c7f67a3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 21 Jun 2022 14:39:28 -0500 Subject: [PATCH 03/30] QQQ-14 fixed checkstyle --- .../backend/module/filesystem/local/FilesystemBackendModule.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java index 25a9e8c7..3a9bca3e 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java @@ -18,6 +18,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + package com.kingsrook.qqq.backend.module.filesystem.local; From 2a2e3370e6914242f3b62a60df24f024710d4f72 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 21 Jun 2022 14:57:42 -0500 Subject: [PATCH 04/30] QQQ-14 update qqq-backend-core snapshot --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2b3fc743..6f7e9a7c 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ com.kingsrook.qqq qqq-backend-core - 0.0.0-20220620.140542-7 + 0.0.0-20220621.195612-8 From ee9f45f8a201f5aa8cb8c01b8eeaf5eb3761778d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 21 Jun 2022 18:43:35 -0500 Subject: [PATCH 05/30] QQQ-14 switch from org.testcontainers/localstack to cloud.localstack (to fix circleci?) --- pom.xml | 18 ++---------- .../module/filesystem/s3/BaseS3Test.java | 28 ++++++------------- 2 files changed, 11 insertions(+), 35 deletions(-) diff --git a/pom.xml b/pom.xml index 6f7e9a7c..db7018f2 100644 --- a/pom.xml +++ b/pom.xml @@ -61,21 +61,9 @@ 1.12.243 - org.testcontainers - testcontainers - 1.17.2 - test - - - org.testcontainers - localstack - 1.17.2 - test - - - org.testcontainers - junit-jupiter - 1.17.2 + cloud.localstack + localstack-utils + 0.2.20 test diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java index 3fb38d3a..8c60963b 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java @@ -22,36 +22,29 @@ package com.kingsrook.qqq.backend.module.filesystem.s3; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; +import cloud.localstack.ServiceName; +import cloud.localstack.awssdkv1.TestUtils; +import cloud.localstack.docker.LocalstackDockerExtension; +import cloud.localstack.docker.annotation.LocalstackDockerProperties; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.testcontainers.containers.localstack.LocalStackContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.utility.DockerImageName; +import org.junit.jupiter.api.extension.ExtendWith; /******************************************************************************* ** *******************************************************************************/ -@Testcontainers +@ExtendWith(LocalstackDockerExtension.class) +@LocalstackDockerProperties(services = { ServiceName.S3 }) public class BaseS3Test { - private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("localstack/localstack"); - public static final String BUCKET_NAME = "localstack-test-bucket"; public static final String TEST_FOLDER = "test-files"; public static final String SUB_FOLDER = "sub-folder"; - @Container - private static LocalStackContainer localStack = new LocalStackContainer(DEFAULT_IMAGE_NAME) - .withServices(LocalStackContainer.Service.S3); - /******************************************************************************* @@ -99,12 +92,7 @@ public class BaseS3Test *******************************************************************************/ protected AmazonS3 getAmazonS3() { - BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(localStack.getAccessKey(), localStack.getSecretKey()); - AmazonS3 amazonS3 = AmazonS3ClientBuilder.standard() - .withEndpointConfiguration(localStack.getEndpointConfiguration(LocalStackContainer.Service.S3)) - .withCredentials(new AWSStaticCredentialsProvider(basicAWSCredentials)) - .build(); - return (amazonS3); + return (TestUtils.getClientS3()); } From 8b3d1ef4252947bcf3b0df507a2202581bb5c0cb Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 21 Jun 2022 18:44:11 -0500 Subject: [PATCH 06/30] QQQ-14 switch from csv to json for the filesystem table --- .../backend/module/filesystem/TestUtils.java | 2 +- .../local/actions/FilesystemActionTest.java | 24 +++++++++---------- .../actions/FilesystemQueryActionTest.java | 2 +- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index c624b4a7..a1f7ae38 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -139,7 +139,7 @@ public class TestUtils .withField(new QFieldMetaData("email", QFieldType.STRING)) .withBackendDetails(new FilesystemTableBackendDetails() .withPath("persons") - .withRecordFormat("csv") + .withRecordFormat("json") .withCardinality("many") ); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java index 7d19512e..39ebf12c 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java @@ -73,22 +73,20 @@ public class FilesystemActionTest } fullPath += File.separatorChar; - String csvHeader = """ - "id","createDate","modifyDate","firstName","lastName","birthDate","email" + String jsonData1 = """ + [ + {"id":1,"createDate":"2021-10-26 14:39:37","modifyDate":"2021-10-26 14:39:37","firstName":"John","lastName":"Doe","birthDate":"1981-01-01","email":"john@kingsrook.com"}, + {"id":2,"createDate":"2022-06-17 14:52:59","modifyDate":"2022-06-17 14:52:59","firstName":"Jane","lastName":"Smith","birthDate":"1982-02-02","email":"jane@kingsrook.com"} + ] """; + FileUtils.writeStringToFile(new File(fullPath + "DATA-1.json"), jsonData1); - String csvData1 = csvHeader + """ - "1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com" - "2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com" + String jsonData2 = """ + [ + {"id":3,"createDate":"2021-11-27 15:40:38","modifyDate":"2021-11-27 15:40:38","firstName":"Homer","lastName":"S","birthDate":"1983-03-03","email":"homer.s@kingsrook.com"} + ] """; - FileUtils.writeStringToFile(new File(fullPath + "DATA-1.csv"), csvData1); - - String csvData2 = csvHeader + """ - "3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com" - "4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com" - "5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com" - """; - FileUtils.writeStringToFile(new File(fullPath + "DATA-2.csv"), csvData2); + FileUtils.writeStringToFile(new File(fullPath + "DATA-2.json"), jsonData2); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java index b918e457..80287082 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java @@ -69,7 +69,7 @@ public class FilesystemQueryActionTest extends FilesystemActionTest { QueryRequest queryRequest = initQueryRequest(); QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest); - Assertions.assertEquals(5, queryResult.getRecords().size(), "Unfiltered query should find all rows"); + Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows"); } From 7b9a0e636d5dc77c535b2231327a28d00f778782 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 21 Jun 2022 18:46:28 -0500 Subject: [PATCH 07/30] QQQ-14 adding localstack orb --- .circleci/config.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ef02745..749480af 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,6 +8,7 @@ executors: orbs: slack: circleci/slack@4.10.1 + localstack: localstack/platform@1.0 commands: run_maven: @@ -39,16 +40,18 @@ commands: jobs: mvn_test: - executor: java17 + executor: localstack/default steps: + - localstack/startup - run_maven: maven_subcommand: test - slack/notify: event: fail mvn_deploy: - executor: java17 + executor: localstack/default steps: + - localstack/startup - run_maven: maven_subcommand: deploy - slack/notify: From 0d402e70c7f3dfce82b9d146d260fdad046493a2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 21 Jun 2022 18:50:26 -0500 Subject: [PATCH 08/30] QQQ-14 try java17 executor with localstack --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 749480af..f5044f7d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,7 +40,7 @@ commands: jobs: mvn_test: - executor: localstack/default + executor: java17 steps: - localstack/startup - run_maven: @@ -49,7 +49,7 @@ jobs: event: fail mvn_deploy: - executor: localstack/default + executor: java17 steps: - localstack/startup - run_maven: From efc78b9d8a9f42ca07d76782450a55eac597009a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 21 Jun 2022 19:10:47 -0500 Subject: [PATCH 09/30] QQQ-14 reverting all localstack changes - does it just work from junit? --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f5044f7d..0ef02745 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,6 @@ executors: orbs: slack: circleci/slack@4.10.1 - localstack: localstack/platform@1.0 commands: run_maven: @@ -42,7 +41,6 @@ jobs: mvn_test: executor: java17 steps: - - localstack/startup - run_maven: maven_subcommand: test - slack/notify: @@ -51,7 +49,6 @@ jobs: mvn_deploy: executor: java17 steps: - - localstack/startup - run_maven: maven_subcommand: deploy - slack/notify: From 00fcc02ed36104ecd7ecb6d56f8c07fc9b011f6f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 21 Jun 2022 19:19:40 -0500 Subject: [PATCH 10/30] QQQ-14 restoring localstack orb --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ef02745..f5044f7d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,6 +8,7 @@ executors: orbs: slack: circleci/slack@4.10.1 + localstack: localstack/platform@1.0 commands: run_maven: @@ -41,6 +42,7 @@ jobs: mvn_test: executor: java17 steps: + - localstack/startup - run_maven: maven_subcommand: test - slack/notify: @@ -49,6 +51,7 @@ jobs: mvn_deploy: executor: java17 steps: + - localstack/startup - run_maven: maven_subcommand: deploy - slack/notify: From 3bd395c90924b406a9cb8af8d1a6892a46ace562 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 21 Jun 2022 19:20:41 -0500 Subject: [PATCH 11/30] QQQ-14 fixed to use executor: localstack/default --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f5044f7d..749480af 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,7 +40,7 @@ commands: jobs: mvn_test: - executor: java17 + executor: localstack/default steps: - localstack/startup - run_maven: @@ -49,7 +49,7 @@ jobs: event: fail mvn_deploy: - executor: java17 + executor: localstack/default steps: - localstack/startup - run_maven: From b88b596e2c5849cb0e8e4b88983df1ba52c5789f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 21 Jun 2022 19:27:36 -0500 Subject: [PATCH 12/30] QQQ-14 adding install_java17 command --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 749480af..ee5bbc20 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,6 +11,12 @@ orbs: localstack: localstack/platform@1.0 commands: + install_java17: + steps: + - run: + name: Install Java 17 + command: | + sudo add-apt-repository -y ppa:openjdk-r/ppa && sudo apt install -y openjdk-17-jdk && sudo rm /etc/alternatives/java && sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java run_maven: parameters: maven_subcommand: @@ -43,6 +49,7 @@ jobs: executor: localstack/default steps: - localstack/startup + - install_java17 - run_maven: maven_subcommand: test - slack/notify: @@ -52,6 +59,7 @@ jobs: executor: localstack/default steps: - localstack/startup + - install_java17 - run_maven: maven_subcommand: deploy - slack/notify: From e90835dcca39344a9b63a4f3b61a67b5ebc69408 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 22 Jun 2022 10:44:12 -0500 Subject: [PATCH 13/30] QQQ-14 attempting to fix circleci localstack failures with randomizePorts --- .../kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java index 8c60963b..45fe56d3 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java @@ -38,7 +38,7 @@ import org.junit.jupiter.api.extension.ExtendWith; ** *******************************************************************************/ @ExtendWith(LocalstackDockerExtension.class) -@LocalstackDockerProperties(services = { ServiceName.S3 }) +@LocalstackDockerProperties(services = { ServiceName.S3 }, randomizePorts = true) public class BaseS3Test { public static final String BUCKET_NAME = "localstack-test-bucket"; From b80b3633aaa3f03379dd5adfe149aa294e2ee3c6 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 22 Jun 2022 10:50:42 -0500 Subject: [PATCH 14/30] QQQ-14 changing to use LocalstackDockerProperties.portEdge instead of the deprecated .randomizePorts --- .../kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java index 45fe56d3..41b3bfee 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java @@ -38,7 +38,7 @@ import org.junit.jupiter.api.extension.ExtendWith; ** *******************************************************************************/ @ExtendWith(LocalstackDockerExtension.class) -@LocalstackDockerProperties(services = { ServiceName.S3 }, randomizePorts = true) +@LocalstackDockerProperties(services = { ServiceName.S3 }, portEdge = "2960") public class BaseS3Test { public static final String BUCKET_NAME = "localstack-test-bucket"; From 26e8bcc14963f56a9573e3df74fad94cc53b6f01 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 22 Jun 2022 11:04:13 -0500 Subject: [PATCH 15/30] QQQ-14 Also adding portElasticSearch, as we still got one port-in-use error w/ just portEdge --- .../kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java index 41b3bfee..aef56d11 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java @@ -38,7 +38,7 @@ import org.junit.jupiter.api.extension.ExtendWith; ** *******************************************************************************/ @ExtendWith(LocalstackDockerExtension.class) -@LocalstackDockerProperties(services = { ServiceName.S3 }, portEdge = "2960") +@LocalstackDockerProperties(services = { ServiceName.S3 }, portEdge = "2960", portElasticSearch = "2961") public class BaseS3Test { public static final String BUCKET_NAME = "localstack-test-bucket"; From ce7b540eb87de6f0ec746ab212b40049d69888d2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 22 Jun 2022 11:20:20 -0500 Subject: [PATCH 16/30] QQQ-14 attempt to cache usr/lib/jvm, to avoid re-installing java17 on each run --- .circleci/config.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ee5bbc20..b116d468 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,8 @@ commands: - run: name: Install Java 17 command: | - sudo add-apt-repository -y ppa:openjdk-r/ppa && sudo apt install -y openjdk-17-jdk && sudo rm /etc/alternatives/java && sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java + if [ ! -e "/usr/lib/jvm/java-17-openjdk-amd64" ]; then sudo add-apt-repository -y ppa:openjdk-r/ppa && sudo apt install -y openjdk-17-jdk fi + sudo rm /etc/alternatives/java && sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java run_maven: parameters: maven_subcommand: @@ -25,6 +26,8 @@ commands: steps: - checkout - restore_cache: + keys: + - usr-lib-jvm keys: - v1-dependencies-{{ checksum "pom.xml" }} - run: @@ -43,6 +46,10 @@ commands: paths: - ~/.m2 key: v1-dependencies-{{ checksum "pom.xml" }} + - save_cache: + paths: + - /usr/lib/jvm/ + key: usr-lib-jvm jobs: mvn_test: From de905b01976a0ec3a2586ca627932b2e5a93b3c6 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 22 Jun 2022 11:21:12 -0500 Subject: [PATCH 17/30] QQQ-14 syntax fix for restore_cache --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b116d468..07eff3b3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,10 +26,11 @@ commands: steps: - checkout - restore_cache: - keys: - - usr-lib-jvm keys: - v1-dependencies-{{ checksum "pom.xml" }} + - restore_cache: + keys: + - usr-lib-jvm - run: name: Run Maven command: | From 685e4d25c3848af12eb971771a37639eded1a201 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 22 Jun 2022 11:22:46 -0500 Subject: [PATCH 18/30] QQQ-14 syntax fix for conditional java-17 install --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 07eff3b3..2d85e1b8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ commands: - run: name: Install Java 17 command: | - if [ ! -e "/usr/lib/jvm/java-17-openjdk-amd64" ]; then sudo add-apt-repository -y ppa:openjdk-r/ppa && sudo apt install -y openjdk-17-jdk fi + if [ ! -e "/usr/lib/jvm/java-17-openjdk-amd64" ]; then sudo add-apt-repository -y ppa:openjdk-r/ppa && sudo apt install -y openjdk-17-jdk; else echo "java 17 already installed"; fi sudo rm /etc/alternatives/java && sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java run_maven: parameters: From 2f383fe3799b4effbaf48d42f73685a8fdcf0f85 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 22 Jun 2022 11:27:51 -0500 Subject: [PATCH 19/30] QQQ-14 add comment explanining deviations from standard config --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2d85e1b8..af5c1807 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,3 +1,6 @@ +## Deviations from qqq-java library standard circleci config: +## - To test AWS S3, uses localstsack executor, orb, and startup step +## - This docker image doesn't have java-17, so we install (and cache) jvm-17 version: 2.1 executors: From de32291fa820e746db480bc88341e3d83e7d3626 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 22 Jun 2022 11:30:34 -0500 Subject: [PATCH 20/30] QQQ-14 moving usr-lib-jvm cache into the install_java17 command --- .circleci/config.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index af5c1807..7743d386 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,11 +16,18 @@ orbs: commands: install_java17: steps: + - restore_cache: + keys: + - usr-lib-jvm - run: name: Install Java 17 command: | if [ ! -e "/usr/lib/jvm/java-17-openjdk-amd64" ]; then sudo add-apt-repository -y ppa:openjdk-r/ppa && sudo apt install -y openjdk-17-jdk; else echo "java 17 already installed"; fi sudo rm /etc/alternatives/java && sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java + - save_cache: + paths: + - /usr/lib/jvm/ + key: usr-lib-jvm run_maven: parameters: maven_subcommand: @@ -31,9 +38,6 @@ commands: - restore_cache: keys: - v1-dependencies-{{ checksum "pom.xml" }} - - restore_cache: - keys: - - usr-lib-jvm - run: name: Run Maven command: | @@ -50,10 +54,6 @@ commands: paths: - ~/.m2 key: v1-dependencies-{{ checksum "pom.xml" }} - - save_cache: - paths: - - /usr/lib/jvm/ - key: usr-lib-jvm jobs: mvn_test: From ed835c9112a8e121d4cb5bec0742213efdf6fc17 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 22 Jun 2022 11:34:53 -0500 Subject: [PATCH 21/30] QQQ-14 give up on caching java-17 (permission errors on restore); attempting resource_class:small for localstack/default executor --- .circleci/config.yml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7743d386..62579ef6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,18 +16,13 @@ orbs: commands: install_java17: steps: - - restore_cache: - keys: - - usr-lib-jvm - run: name: Install Java 17 command: | - if [ ! -e "/usr/lib/jvm/java-17-openjdk-amd64" ]; then sudo add-apt-repository -y ppa:openjdk-r/ppa && sudo apt install -y openjdk-17-jdk; else echo "java 17 already installed"; fi - sudo rm /etc/alternatives/java && sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java - - save_cache: - paths: - - /usr/lib/jvm/ - key: usr-lib-jvm + sudo add-apt-repository -y ppa:openjdk-r/ppa + sudo apt install -y openjdk-17-jdk + sudo rm /etc/alternatives/java + sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java run_maven: parameters: maven_subcommand: @@ -57,7 +52,9 @@ commands: jobs: mvn_test: - executor: localstack/default + executor: + name: localstack/default + resource_class: small steps: - localstack/startup - install_java17 @@ -67,7 +64,9 @@ jobs: event: fail mvn_deploy: - executor: localstack/default + executor: + name: localstack/default + resource_class: small steps: - localstack/startup - install_java17 From 5aa0aeecf6877fd734e15ad8cda2c82b9dbb7300 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 22 Jun 2022 11:37:08 -0500 Subject: [PATCH 22/30] QQQ-14 revert adding resource_class:small to the localstack/default executor --- .circleci/config.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 62579ef6..40a1a443 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,9 +52,7 @@ commands: jobs: mvn_test: - executor: - name: localstack/default - resource_class: small + executor: localstack/default steps: - localstack/startup - install_java17 @@ -64,9 +62,7 @@ jobs: event: fail mvn_deploy: - executor: - name: localstack/default - resource_class: small + executor: localstack/default steps: - localstack/startup - install_java17 From 62b015200a93d054908c6886908dc44bd18ad708 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 23 Jun 2022 14:50:24 -0500 Subject: [PATCH 23/30] QQQ-16 updating version of qqq-backend-core --- pom.xml | 2 +- .../filesystem/local/actions/FilesystemInsertAction.java | 4 ++-- .../filesystem/local/actions/FilesystemUpdateAction.java | 6 +++--- .../module/filesystem/s3/actions/S3InsertAction.java | 4 ++-- .../module/filesystem/s3/actions/S3UpdateAction.java | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index db7018f2..018c2f86 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ com.kingsrook.qqq qqq-backend-core - 0.0.0-20220621.195612-8 + 0.0.0-20220623.193535-9 diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java index 328d7b43..c9130de2 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java @@ -27,7 +27,7 @@ import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest; import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult; -import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; import org.apache.commons.lang.NotImplementedException; @@ -49,7 +49,7 @@ public class FilesystemInsertAction implements InsertInterface InsertResult rs = new InsertResult(); QTableMetaData table = insertRequest.getTable(); - List recordsWithStatus = new ArrayList<>(); + List recordsWithStatus = new ArrayList<>(); rs.setRecords(recordsWithStatus); throw new NotImplementedException("Filesystem insert not implemented"); diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java index bae14d1b..a81c96d8 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java @@ -27,7 +27,7 @@ import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest; import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult; -import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; import org.apache.commons.lang.NotImplementedException; @@ -49,8 +49,8 @@ public class FilesystemUpdateAction implements UpdateInterface UpdateResult rs = new UpdateResult(); QTableMetaData table = updateRequest.getTable(); - List recordsWithStatus = new ArrayList<>(); - rs.setRecords(recordsWithStatus); + List records = new ArrayList<>(); + rs.setRecords(records); throw new NotImplementedException("Filesystem update not implemented"); diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java index 310709b1..b06ddc29 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java @@ -27,7 +27,7 @@ import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest; import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult; -import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; import org.apache.commons.lang.NotImplementedException; @@ -49,7 +49,7 @@ public class S3InsertAction implements InsertInterface InsertResult rs = new InsertResult(); QTableMetaData table = insertRequest.getTable(); - List recordsWithStatus = new ArrayList<>(); + List recordsWithStatus = new ArrayList<>(); rs.setRecords(recordsWithStatus); throw new NotImplementedException("S3 insert not implemented"); diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java index 31844f47..9dd5364b 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java @@ -27,7 +27,7 @@ import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest; import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult; -import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; import org.apache.commons.lang.NotImplementedException; @@ -49,8 +49,8 @@ public class S3UpdateAction implements UpdateInterface UpdateResult rs = new UpdateResult(); QTableMetaData table = updateRequest.getTable(); - List recordsWithStatus = new ArrayList<>(); - rs.setRecords(recordsWithStatus); + List records = new ArrayList<>(); + rs.setRecords(records); throw new NotImplementedException("S3 update not implemented"); From 93a06cf1ab92054420e896e9718b721815c89ca3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 23 Jun 2022 14:53:06 -0500 Subject: [PATCH 24/30] QQQ-14, QQQ-16 updated filesystem, s3 implementation, including passable version of basic ETL process & cleanup --- .../FilesystemBackendModuleInterface.java | 51 +++++ .../FilesystemRecordBackendDetailFields.java | 32 ++++ .../actions/AbstractBaseFilesystemAction.java | 6 + .../exceptions/FilesystemException.java | 52 ++++++ .../local/FilesystemBackendModule.java | 78 +++++++- .../actions/AbstractFilesystemAction.java | 26 ++- .../BasicETLCleanupSourceFilesFunction.java | 110 +++++++++++ .../module/filesystem/s3/S3BackendModule.java | 74 +++++++- .../s3/actions/AbstractS3Action.java | 17 ++ .../module/filesystem/s3/utils/S3Utils.java | 51 ++++- .../backend/module/filesystem/TestUtils.java | 15 +- .../local/FilesystemBackendModuleTest.java | 174 ++++++++++++++++++ .../local/actions/FilesystemActionTest.java | 4 +- .../actions/FilesystemQueryActionTest.java | 4 + .../module/filesystem/s3/BaseS3Test.java | 2 +- .../filesystem/s3/S3BackendModuleTest.java | 151 +++++++++++++++ .../s3/actions/S3QueryActionTest.java | 8 +- 17 files changed, 840 insertions(+), 15 deletions(-) create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemRecordBackendDetailFields.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/exceptions/FilesystemException.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java create mode 100644 src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java create mode 100644 src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java new file mode 100644 index 00000000..9a7578a4 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java @@ -0,0 +1,51 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base; + + +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; + + +/******************************************************************************* + ** Interface to add additional functionality commonly among the various filesystem + ** module implementations. + *******************************************************************************/ +public interface FilesystemBackendModuleInterface +{ + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + *******************************************************************************/ + void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException; + + /******************************************************************************* + ** Move a file from a source path, to a destination path. + ** + ** @throws FilesystemException if the move is known to have failed + *******************************************************************************/ + void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException; +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemRecordBackendDetailFields.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemRecordBackendDetailFields.java new file mode 100644 index 00000000..415aa06f --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemRecordBackendDetailFields.java @@ -0,0 +1,32 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base; + + +/******************************************************************************* + ** Define the field names (keys) to be used in the backendDetails structure + ** under Records from this (these) modules. + *******************************************************************************/ +public interface FilesystemRecordBackendDetailFields +{ + String FULL_PATH = "fullPath"; +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 07b5f3dc..b8e07308 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -59,6 +59,10 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ public abstract InputStream readFile(FILE file) throws IOException; + /******************************************************************************* + ** Add backend details to records about the file that they are in. + *******************************************************************************/ + protected abstract void addBackendDetailsToRecords(List recordsInFile, FILE file); /******************************************************************************* @@ -133,6 +137,7 @@ public abstract class AbstractBaseFilesystemAction { String fileContents = IOUtils.toString(readFile(file)); List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); records.addAll(recordsInFile); break; @@ -141,6 +146,7 @@ public abstract class AbstractBaseFilesystemAction { String fileContents = IOUtils.toString(readFile(file)); List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); records.addAll(recordsInFile); break; diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/exceptions/FilesystemException.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/exceptions/FilesystemException.java new file mode 100644 index 00000000..cbda757b --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/exceptions/FilesystemException.java @@ -0,0 +1,52 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.exceptions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; + + +/******************************************************************************* + ** Checked exception to be thrown from actions within this module. + *******************************************************************************/ +public class FilesystemException extends QException +{ + /******************************************************************************* + ** Constructor of message + ** + *******************************************************************************/ + public FilesystemException(String message) + { + super(message); + } + + + + /******************************************************************************* + ** Constructor of message & cause + ** + *******************************************************************************/ + public FilesystemException(String message, Throwable cause) + { + super(message, cause); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java index 3a9bca3e..09594b99 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java @@ -22,26 +22,34 @@ package com.kingsrook.qqq.backend.module.filesystem.local; +import java.io.File; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemUpdateAction; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* ** QQQ Backend module for working with (local) Filesystems. *******************************************************************************/ -public class FilesystemBackendModule implements QBackendModuleInterface +public class FilesystemBackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface { + private static final Logger LOG = LogManager.getLogger(FilesystemBackendModule.class); /******************************************************************************* @@ -65,6 +73,7 @@ public class FilesystemBackendModule implements QBackendModuleInterface } + /******************************************************************************* ** Method to identify the class used for table-backend details for this module. *******************************************************************************/ @@ -117,4 +126,71 @@ public class FilesystemBackendModule implements QBackendModuleInterface { return (new FilesystemDeleteAction()); } + + + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + *******************************************************************************/ + @Override + public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + { + File file = new File(fileReference); + if(!file.exists()) + { + ////////////////////////////////////////////////////////////////////////////////////////////// + // if the file doesn't exist, just exit with noop. don't throw an error - that should only // + // happen if the "contract" of the method is broken, and the file still exists // + ////////////////////////////////////////////////////////////////////////////////////////////// + LOG.debug("Not deleting file [{}], because it does not exist.", file); + return; + } + + if(!file.delete()) + { + throw (new FilesystemException("Failed to delete file: " + fileReference)); + } + } + + + + /******************************************************************************* + ** Move a file from a source path, to a destination path. + ** + ** @throws FilesystemException if the delete is known to have failed + *******************************************************************************/ + @Override + public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException + { + File sourceFile = new File(source); + File destinationFile = new File(destination); + File destinationParent = destinationFile.getParentFile(); + + if(!sourceFile.exists()) + { + throw (new FilesystemException("Cannot move file " + source + ", as it does not exist.")); + } + + ////////////////////////////////////////////////////////////////////////////////////// + // if the destination folder doesn't exist, try to make it - and fail if that fails // + ////////////////////////////////////////////////////////////////////////////////////// + if(!destinationParent.exists()) + { + LOG.debug("Making destination directory {} for move", destinationParent.getAbsolutePath()); + if(!destinationParent.mkdirs()) + { + throw (new FilesystemException("Failed to make destination directory " + destinationParent.getAbsolutePath() + " to move " + source + " into.")); + } + } + + if(!sourceFile.renameTo(destinationFile)) + { + throw (new FilesystemException("Failed to move (rename) file " + source + " to " + destination)); + } + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index fde5c1d5..91b5654b 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -27,9 +27,12 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; @@ -47,7 +50,14 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction { String fullPath = getFullPath(table, backendBase); File directory = new File(fullPath); - return Arrays.asList(directory.listFiles()); + File[] files = directory.listFiles(); + + if(files == null) + { + return Collections.emptyList(); + } + + return (Arrays.stream(files).filter(File::isFile).toList()); } @@ -61,4 +71,18 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction return (new FileInputStream(file)); } + + + /******************************************************************************* + ** Add backend details to records about the file that they are in. + *******************************************************************************/ + @Override + protected void addBackendDetailsToRecords(List recordsInFile, File file) + { + recordsInFile.forEach(record -> + { + record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, file.getAbsolutePath()); + }); + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java new file mode 100644 index 00000000..f9a6a26f --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java @@ -0,0 +1,110 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic; + + +import java.io.File; +import java.util.Set; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; + + +/******************************************************************************* + ** Function body for performing the Cleanup step of a basic ETL process - e.g., + ** after the loading, delete or move the processed file(s). + *******************************************************************************/ +public class BasicETLCleanupSourceFilesFunction implements FunctionBody +{ + @Override + public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + { + String sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE); + QTableMetaData table = runFunctionRequest.getInstance().getTable(sourceTableName); + QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(table.getBackendName()); + if(!(module instanceof FilesystemBackendModuleInterface filesystemModule)) + { + throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function.")); + } + + Set sourceFiles = runFunctionRequest.getRecords().stream() + .map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH)) + .collect(Collectors.toSet()); + for(String sourceFile : sourceFiles) + { + String moveOrDelete = runFunctionRequest.getValueString("moveOrDelete"); + if("delete".equals(moveOrDelete)) + { + filesystemModule.deleteFile(runFunctionRequest.getInstance(), table, sourceFile); + } + else if("move".equals(moveOrDelete)) + { + String destinationForMoves = runFunctionRequest.getValueString("destinationForMoves"); + if(!StringUtils.hasContent(destinationForMoves)) + { + throw (new QException("Field [destinationForMoves] is missing a value.")); + } + File file = new File(sourceFile); + String destinationPath = destinationForMoves + File.separator + file.getName(); + filesystemModule.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath); + } + else + { + throw (new QException("Unexpected value [" + moveOrDelete + "] for field [moveOrDelete]. Must be either [move] or [delete].")); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QFunctionMetaData defineFunctionMetaData() + { + return (new QFunctionMetaData() + .withName("cleanupSourceFiles") + .withCode(new QCodeReference() + .withName(this.getClass().getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.FUNCTION)) + .withInputData(new QFunctionInputMetaData() + .addField(new QFieldMetaData("moveOrDelete", QFieldType.STRING)) + .addField(new QFieldMetaData("destinationForMoves", QFieldType.STRING)))); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java index 7c574cf3..33bffb0b 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java @@ -23,25 +23,32 @@ package com.kingsrook.qqq.backend.module.filesystem.s3; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; /******************************************************************************* ** QQQ Backend module for working with AWS S3 filesystems *******************************************************************************/ -public class S3BackendModule implements QBackendModuleInterface +public class S3BackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface { + private S3Utils s3Utils; + /******************************************************************************* @@ -72,7 +79,44 @@ public class S3BackendModule implements QBackendModuleInterface @Override public Class getTableBackendDetailsClass() { - return S3TableBackendDetails.class; + return (S3TableBackendDetails.class); + } + + + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + *******************************************************************************/ + @Override + public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + { + QBackendMetaData backend = instance.getBackend(table.getBackendName()); + String bucketName = ((S3BackendMetaData) backend).getBucketName(); + + getS3Utils().deleteObject(bucketName, fileReference); + } + + + + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the move is known to have failed + *******************************************************************************/ + @Override + public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException + { + QBackendMetaData backend = instance.getBackend(table.getBackendName()); + String bucketName = ((S3BackendMetaData) backend).getBucketName(); + + getS3Utils().moveObject(bucketName, source, destination); } @@ -118,4 +162,30 @@ public class S3BackendModule implements QBackendModuleInterface { return (new S3DeleteAction()); } + + + + /******************************************************************************* + ** Setter for s3Utils + *******************************************************************************/ + public void setS3Utils(S3Utils s3Utils) + { + this.s3Utils = s3Utils; + } + + + + /******************************************************************************* + ** Internal accessor for the s3Utils object - should always use this, not the field. + *******************************************************************************/ + private S3Utils getS3Utils() + { + if(s3Utils == null) + { + s3Utils = new S3Utils(); + } + + return s3Utils; + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 12c69959..1fb11d34 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -26,8 +26,10 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; @@ -96,4 +98,19 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction recordsInFile, S3ObjectSummary s3ObjectSummary) + { + recordsInFile.forEach(record -> + { + record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, s3ObjectSummary.getKey()); + }); + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java index 4dfd4f1c..65a25367 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java @@ -31,12 +31,17 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.ListObjectsV2Request; import com.amazonaws.services.s3.model.ListObjectsV2Result; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; /******************************************************************************* ** Utility methods for working with AWS S3. + ** + ** Note: May need a constructor (or similar) in the future that takes the + ** S3BackendMetaData - e.g., if we need some metaData to construct the AmazonS3 + ** (api client) object, such as region, or authentication. *******************************************************************************/ public class S3Utils { @@ -111,8 +116,9 @@ public class S3Utils } + /******************************************************************************* - ** + ** Get the contents (as an InputStream) for an object in s3 *******************************************************************************/ public InputStream getObjectAsInputStream(S3ObjectSummary s3ObjectSummary) { @@ -120,6 +126,48 @@ public class S3Utils } + + /******************************************************************************* + ** Delete an object (file) from a bucket + *******************************************************************************/ + public void deleteObject(String bucketName, String key) throws FilesystemException + { + ////////////////////////////////////////////////////////////////////////////////////////////// + // note, aws s3 api does not appear to have any way to check the success or failure here... // + ////////////////////////////////////////////////////////////////////////////////////////////// + try + { + getS3().deleteObject(bucketName, key); + } + catch(Exception e) + { + throw (new FilesystemException("Error deleting s3 object " + key + " in bucket " + bucketName, e)); + } + } + + + + /******************************************************************************* + ** Move an object (file) within a bucket + *******************************************************************************/ + public void moveObject(String bucketName, String source, String destination) throws FilesystemException + { + ////////////////////////////////////////////////////////////////////////////////////////////// + // note, aws s3 api does not appear to have any way to check the success or failure here... // + ////////////////////////////////////////////////////////////////////////////////////////////// + try + { + getS3().copyObject(bucketName, source, bucketName, destination); + getS3().deleteObject(bucketName, source); + } + catch(Exception e) + { + throw (new FilesystemException("Error moving s3 object " + source + " to " + destination + " in bucket " + bucketName, e)); + } + } + + + /******************************************************************************* ** Setter for AmazonS3 client object. *******************************************************************************/ @@ -143,5 +191,4 @@ public class S3Utils return s3; } - } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index a1f7ae38..17e7f266 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -43,8 +43,15 @@ import org.apache.commons.io.FileUtils; *******************************************************************************/ public class TestUtils { - private static int testInstanceCounter = 0; - private final static String BASE_PATH = "/tmp/filesystem-tests"; + public static final String TABLE_NAME_PERSON = "person"; + public static final String TABLE_NAME_PERSON_S3 = "person-s3"; + + /////////////////////////////////////////////////////////////////// + // shouldn't be accessed directly, as we append a counter to it. // + /////////////////////////////////////////////////////////////////// + public static final String BASE_PATH = "/tmp/filesystem-tests"; + + private static int testInstanceCounter = 0; @@ -126,7 +133,7 @@ public class TestUtils public static QTableMetaData defineLocalFilesystemCSVPersonTable() { return new QTableMetaData() - .withName("person") + .withName(TABLE_NAME_PERSON) .withLabel("Person") .withBackendName(defineLocalFilesystemBackend().getName()) .withPrimaryKeyField("id") @@ -165,7 +172,7 @@ public class TestUtils public static QTableMetaData defineS3CSVPersonTable() { return new QTableMetaData() - .withName("person-s3") + .withName(TABLE_NAME_PERSON_S3) .withLabel("Person S3 Table") .withBackendName(defineS3Backend().getName()) .withPrimaryKeyField("id") diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java new file mode 100644 index 00000000..d6e1e681 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java @@ -0,0 +1,174 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local; + + +import java.io.File; +import java.io.IOException; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for FilesystemBackendModule + *******************************************************************************/ +public class FilesystemBackendModuleTest +{ + private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist"; + + + + @BeforeEach + public void beforeEach() throws IOException + { + new FilesystemActionTest().primeFilesystem(); + } + + + + @AfterEach + public void afterEach() throws Exception + { + new FilesystemActionTest().cleanFilesystem(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteFile() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // first list the files - then delete one, then re-list, and assert that we have one fewer // + ///////////////////////////////////////////////////////////////////////////////////////////// + List filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + + FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); + filesystemBackendModule.deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath()); + + List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + Assertions.assertEquals(filesBeforeDelete.size() - 1, filesAfterDelete.size(), + "Should be one fewer file listed after deleting one."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteFileDoesNotExist() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count // + // note, we'd like to detect the non-delete, but there's no such info back from aws it appears? // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + + FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); + filesystemBackendModule.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + + List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + Assertions.assertEquals(filesBeforeDelete.size(), filesAfterDelete.size(), + "Should be same number of files after deleting bogus path"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMoveFile() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON); + String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath(); + String subPath = basePath + File.separator + "subdir"; + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // first list the files (non-recursively) - then move one into a sub-folder, then re-list, and // + // assert that we have one fewer then list again including sub-folders, and see the changed count // + //////////////////////////////////////////////////////////////////////////////////////////////////// + List filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + + FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); + String originalFilePath = filesBeforeMove.get(0).getAbsolutePath(); + String movedFilePath = originalFilePath.replace(basePath, subPath); + + filesystemBackendModule.moveFile(qInstance, table, originalFilePath, movedFilePath); + + List filesAfterMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + Assertions.assertEquals(filesBeforeMove.size() - 1, filesAfterMove.size(), "Should be one fewer file in the listing after moving one."); + + ////////////////////////////////////////////////////////////////////////// + // move the file back and assert that the count goes back to the before // + ////////////////////////////////////////////////////////////////////////// + filesystemBackendModule.moveFile(qInstance, table, movedFilePath, originalFilePath); + + List filesAfterMoveBack = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + Assertions.assertEquals(filesBeforeMove.size(), filesAfterMoveBack.size(), "Should be original number of files after moving back"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMoveFileDoesNotExit() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON); + String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath(); + String subPath = basePath + File.separator + "subdir"; + List filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); + String originalFilePath = filesBeforeMove.get(0).getAbsolutePath(); + String movedFilePath = originalFilePath.replace(basePath, subPath); + + Assertions.assertThrows(FilesystemException.class, () -> + { + FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); + filesystemBackendModule.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, movedFilePath); + }); + + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java index 39ebf12c..9e9b38a2 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java @@ -43,7 +43,7 @@ public class FilesystemActionTest /******************************************************************************* ** Set up the file system *******************************************************************************/ - protected void primeFilesystem() throws IOException + public void primeFilesystem() throws IOException { TestUtils.cleanInstanceFiles(); TestUtils.increaseTestInstanceCounter(); @@ -94,7 +94,7 @@ public class FilesystemActionTest /******************************************************************************* ** *******************************************************************************/ - protected void cleanFilesystem() throws IOException + public void cleanFilesystem() throws IOException { TestUtils.cleanInstanceFiles(); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java index 80287082..f624cb5a 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java @@ -27,6 +27,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -70,6 +71,9 @@ public class FilesystemQueryActionTest extends FilesystemActionTest QueryRequest queryRequest = initQueryRequest(); QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest); Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows"); + Assertions.assertTrue(queryResult.getRecords().stream() + .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)), + "All records should have a full-path in their backend details, matching the test folder name"); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java index aef56d11..3563d91c 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java @@ -35,7 +35,7 @@ import org.junit.jupiter.api.extension.ExtendWith; /******************************************************************************* - ** + ** Base class for tests that want to be able to work with localstack s3. *******************************************************************************/ @ExtendWith(LocalstackDockerExtension.class) @LocalstackDockerProperties(services = { ServiceName.S3 }, portEdge = "2960", portElasticSearch = "2961") diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java new file mode 100644 index 00000000..e0cb0489 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java @@ -0,0 +1,151 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3; + + +import java.util.List; +import java.util.UUID; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for S3BackendModule + *******************************************************************************/ +public class S3BackendModuleTest extends BaseS3Test +{ + private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteFile() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // first list the files - then delete one, then re-list, and assert that we have one fewer // + ///////////////////////////////////////////////////////////////////////////////////////////// + List s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + + S3BackendModule s3BackendModule = new S3BackendModule(); + s3BackendModule.setS3Utils(getS3Utils()); + s3BackendModule.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey()); + + List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(), + "Should be one fewer file listed after deleting one."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteFileDoesNotExist() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count // + // note, we'd like to detect the non-delete, but there's no such info back from aws it appears? // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + + S3BackendModule s3BackendModule = new S3BackendModule(); + s3BackendModule.setS3Utils(getS3Utils()); + s3BackendModule.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + + List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size(), s3ObjectSummariesAfterDelete.size(), + "Should be same number of files after deleting bogus path"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMoveFile() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3); + String subPath = TEST_FOLDER + "/" + SUB_FOLDER; + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // first list the files (non-recursively) - then move one into a sub-folder, then re-list, and // + // assert that we have one fewer then list again including sub-folders, and see the changed count // + //////////////////////////////////////////////////////////////////////////////////////////////////// + List s3ObjectSummariesBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + List s3ObjectSummariesInSubFolderBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, subPath, false); + List s3ObjectSummariesRecursiveBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, true); + + S3BackendModule s3BackendModule = new S3BackendModule(); + s3BackendModule.setS3Utils(getS3Utils()); + String key = s3ObjectSummariesBeforeMove.get(0).getKey(); + s3BackendModule.moveFile(qInstance, table, key, key.replaceFirst(TEST_FOLDER, subPath)); + + List s3ObjectSummariesAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + List s3ObjectSummariesRecursiveAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, true); + List s3ObjectSummariesInSubFolderAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, subPath, false); + + Assertions.assertEquals(s3ObjectSummariesBeforeMove.size() - 1, s3ObjectSummariesAfterMove.size(), + "Should be one fewer file in the non-recursive listing after moving one."); + Assertions.assertEquals(s3ObjectSummariesRecursiveBeforeMove.size(), s3ObjectSummariesRecursiveAfterMove.size(), + "Should be same number of files in the recursive listing before and after the move"); + Assertions.assertEquals(s3ObjectSummariesInSubFolderBeforeMove.size() + 1, s3ObjectSummariesInSubFolderAfterMove.size(), + "Should be one move file in the sub-folder listing after moving one."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMoveFileDoesNotExit() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3); + String subPath = TEST_FOLDER + "/" + SUB_FOLDER; + + S3BackendModule s3BackendModule = new S3BackendModule(); + s3BackendModule.setS3Utils(getS3Utils()); + + Assertions.assertThrows(FilesystemException.class, () -> + s3BackendModule.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, subPath + "/" + UUID.randomUUID()) + ); + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java index 6fe9d2e0..ae490a44 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java @@ -27,6 +27,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -44,11 +45,14 @@ public class S3QueryActionTest extends BaseS3Test @Test public void testQuery1() throws QException { - QueryRequest queryRequest = initQueryRequest(); + QueryRequest queryRequest = initQueryRequest(); S3QueryAction s3QueryAction = new S3QueryAction(); s3QueryAction.setS3Utils(getS3Utils()); - QueryResult queryResult = s3QueryAction.execute(queryRequest); + QueryResult queryResult = s3QueryAction.execute(queryRequest); Assertions.assertEquals(5, queryResult.getRecords().size(), "Expected # of rows from unfiltered query"); + Assertions.assertTrue(queryResult.getRecords().stream() + .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseS3Test.TEST_FOLDER)), + "All records should have a full-path in their backend details, matching the test folder name"); } From daacfa684955dbd9453ce2d50deacf7783beac4d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 23 Jun 2022 16:49:17 -0500 Subject: [PATCH 25/30] QQQ-16 increasing test coverage --- pom.xml | 2 +- .../local/actions/FilesystemDeleteAction.java | 5 +- .../local/actions/FilesystemInsertAction.java | 9 +- .../local/actions/FilesystemUpdateAction.java | 8 +- .../BasicETLCleanupSourceFilesFunction.java | 18 ++- .../filesystem/s3/actions/S3DeleteAction.java | 5 +- .../filesystem/s3/actions/S3InsertAction.java | 8 +- .../filesystem/s3/actions/S3UpdateAction.java | 9 +- .../backend/module/filesystem/TestUtils.java | 39 ++++- .../local/FilesystemBackendModuleTest.java | 8 +- ...asicETLCleanupSourceFilesFunctionTest.java | 143 ++++++++++++++++++ 11 files changed, 213 insertions(+), 41 deletions(-) create mode 100644 src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java diff --git a/pom.xml b/pom.xml index 018c2f86..2d069357 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ com.kingsrook.qqq qqq-backend-core - 0.0.0-20220623.193535-9 + 0.0.0-20220623.214704-10 diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java index 4d6d9f92..9843bb4e 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java @@ -25,7 +25,6 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest; import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; import org.apache.commons.lang.NotImplementedException; @@ -41,12 +40,13 @@ public class FilesystemDeleteAction implements DeleteInterface *******************************************************************************/ public DeleteResult execute(DeleteRequest deleteRequest) throws QException { + throw new NotImplementedException("Filesystem delete not implemented"); + /* try { DeleteResult rs = new DeleteResult(); QTableMetaData table = deleteRequest.getTable(); - throw new NotImplementedException("Filesystem delete not implemented"); // return rs; } @@ -54,6 +54,7 @@ public class FilesystemDeleteAction implements DeleteInterface { throw new QException("Error executing delete: " + e.getMessage(), e); } + */ } } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java index c9130de2..9a8ca832 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java @@ -22,13 +22,9 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions; -import java.util.ArrayList; -import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest; import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; import org.apache.commons.lang.NotImplementedException; @@ -44,6 +40,8 @@ public class FilesystemInsertAction implements InsertInterface *******************************************************************************/ public InsertResult execute(InsertRequest insertRequest) throws QException { + throw new NotImplementedException("Filesystem insert not implemented"); + /* try { InsertResult rs = new InsertResult(); @@ -52,14 +50,13 @@ public class FilesystemInsertAction implements InsertInterface List recordsWithStatus = new ArrayList<>(); rs.setRecords(recordsWithStatus); - throw new NotImplementedException("Filesystem insert not implemented"); - // return rs; } catch(Exception e) { throw new QException("Error executing insert: " + e.getMessage(), e); } + */ } } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java index a81c96d8..1f762164 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java @@ -22,13 +22,9 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions; -import java.util.ArrayList; -import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest; import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; import org.apache.commons.lang.NotImplementedException; @@ -44,6 +40,8 @@ public class FilesystemUpdateAction implements UpdateInterface *******************************************************************************/ public UpdateResult execute(UpdateRequest updateRequest) throws QException { + throw new NotImplementedException("Filesystem update not implemented"); + /* try { UpdateResult rs = new UpdateResult(); @@ -52,7 +50,6 @@ public class FilesystemUpdateAction implements UpdateInterface List records = new ArrayList<>(); rs.setRecords(records); - throw new NotImplementedException("Filesystem update not implemented"); // return rs; } @@ -60,6 +57,7 @@ public class FilesystemUpdateAction implements UpdateInterface { throw new QException("Error executing update: " + e.getMessage(), e); } + */ } } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java index f9a6a26f..e35ad59b 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; @@ -51,12 +52,19 @@ import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendD *******************************************************************************/ public class BasicETLCleanupSourceFilesFunction implements FunctionBody { + public static final String FIELD_MOVE_OR_DELETE = "moveOrDelete"; + public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves"; + + + @Override public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException { String sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE); QTableMetaData table = runFunctionRequest.getInstance().getTable(sourceTableName); - QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(table.getBackendName()); + QBackendMetaData backend = runFunctionRequest.getInstance().getBackendForTable(sourceTableName); + QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend); + if(!(module instanceof FilesystemBackendModuleInterface filesystemModule)) { throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function.")); @@ -67,17 +75,17 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody .collect(Collectors.toSet()); for(String sourceFile : sourceFiles) { - String moveOrDelete = runFunctionRequest.getValueString("moveOrDelete"); + String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE); if("delete".equals(moveOrDelete)) { filesystemModule.deleteFile(runFunctionRequest.getInstance(), table, sourceFile); } else if("move".equals(moveOrDelete)) { - String destinationForMoves = runFunctionRequest.getValueString("destinationForMoves"); + String destinationForMoves = runFunctionRequest.getValueString(FIELD_DESTINATION_FOR_MOVES); if(!StringUtils.hasContent(destinationForMoves)) { - throw (new QException("Field [destinationForMoves] is missing a value.")); + throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value.")); } File file = new File(sourceFile); String destinationPath = destinationForMoves + File.separator + file.getName(); @@ -85,7 +93,7 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody } else { - throw (new QException("Unexpected value [" + moveOrDelete + "] for field [moveOrDelete]. Must be either [move] or [delete].")); + throw (new QException("Unexpected value [" + moveOrDelete + "] for field [" + FIELD_MOVE_OR_DELETE + "]. Must be either [move] or [delete].")); } } } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java index 8598c10e..2a4a8448 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java @@ -25,7 +25,6 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest; import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; import org.apache.commons.lang.NotImplementedException; @@ -41,12 +40,13 @@ public class S3DeleteAction implements DeleteInterface *******************************************************************************/ public DeleteResult execute(DeleteRequest deleteRequest) throws QException { + throw new NotImplementedException("S3 delete not implemented"); + /* try { DeleteResult rs = new DeleteResult(); QTableMetaData table = deleteRequest.getTable(); - throw new NotImplementedException("S3 delete not implemented"); // return rs; } @@ -54,6 +54,7 @@ public class S3DeleteAction implements DeleteInterface { throw new QException("Error executing delete: " + e.getMessage(), e); } + */ } } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java index b06ddc29..caeed82b 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java @@ -22,13 +22,9 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; -import java.util.ArrayList; -import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest; import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; import org.apache.commons.lang.NotImplementedException; @@ -44,6 +40,8 @@ public class S3InsertAction implements InsertInterface *******************************************************************************/ public InsertResult execute(InsertRequest insertRequest) throws QException { + throw new NotImplementedException("S3 insert not implemented"); + /* try { InsertResult rs = new InsertResult(); @@ -52,7 +50,6 @@ public class S3InsertAction implements InsertInterface List recordsWithStatus = new ArrayList<>(); rs.setRecords(recordsWithStatus); - throw new NotImplementedException("S3 insert not implemented"); // return rs; } @@ -60,6 +57,7 @@ public class S3InsertAction implements InsertInterface { throw new QException("Error executing insert: " + e.getMessage(), e); } + */ } } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java index 9dd5364b..4955ffb2 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java @@ -22,13 +22,9 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; -import java.util.ArrayList; -import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest; import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; import org.apache.commons.lang.NotImplementedException; @@ -44,6 +40,8 @@ public class S3UpdateAction implements UpdateInterface *******************************************************************************/ public UpdateResult execute(UpdateRequest updateRequest) throws QException { + throw new NotImplementedException("S3 update not implemented"); + /* try { UpdateResult rs = new UpdateResult(); @@ -52,14 +50,13 @@ public class S3UpdateAction implements UpdateInterface List records = new ArrayList<>(); rs.setRecords(records); - throw new NotImplementedException("S3 update not implemented"); - // return rs; } catch(Exception e) { throw new QException("Error executing update: " + e.getMessage(), e); } + */ } } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index 17e7f266..d94da070 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -26,10 +26,13 @@ import java.io.File; import java.io.IOException; import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.mock.MockAuthenticationModule; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; @@ -43,8 +46,10 @@ import org.apache.commons.io.FileUtils; *******************************************************************************/ public class TestUtils { - public static final String TABLE_NAME_PERSON = "person"; - public static final String TABLE_NAME_PERSON_S3 = "person-s3"; + public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; + public static final String BACKEND_NAME_S3 = "s3"; + public static final String TABLE_NAME_PERSON_LOCAL_FS = "person"; + public static final String TABLE_NAME_PERSON_S3 = "person-s3"; /////////////////////////////////////////////////////////////////// // shouldn't be accessed directly, as we append a counter to it. // @@ -103,6 +108,7 @@ public class TestUtils public static QInstance defineInstance() throws QInstanceValidationException { QInstance qInstance = new QInstance(); + qInstance.setAuthentication(defineAuthentication()); qInstance.addBackend(defineLocalFilesystemBackend()); qInstance.addTable(defineLocalFilesystemCSVPersonTable()); qInstance.addBackend(defineS3Backend()); @@ -115,6 +121,19 @@ public class TestUtils + /******************************************************************************* + ** Define the authentication used in standard tests - using 'mock' type. + ** + *******************************************************************************/ + public static QAuthenticationMetaData defineAuthentication() + { + return new QAuthenticationMetaData() + .withName("mock") + .withType("mock"); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -122,7 +141,7 @@ public class TestUtils { return (new FilesystemBackendMetaData() .withBasePath(BASE_PATH + File.separator + testInstanceCounter) - .withName("local-filesystem")); + .withName(BACKEND_NAME_LOCAL_FS)); } @@ -133,7 +152,7 @@ public class TestUtils public static QTableMetaData defineLocalFilesystemCSVPersonTable() { return new QTableMetaData() - .withName(TABLE_NAME_PERSON) + .withName(TABLE_NAME_PERSON_LOCAL_FS) .withLabel("Person") .withBackendName(defineLocalFilesystemBackend().getName()) .withPrimaryKeyField("id") @@ -161,7 +180,7 @@ public class TestUtils return (new S3BackendMetaData() .withBucketName(BaseS3Test.BUCKET_NAME) .withBasePath(BaseS3Test.TEST_FOLDER) - .withName("s3")); + .withName(BACKEND_NAME_S3)); } @@ -189,4 +208,14 @@ public class TestUtils ); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QSession getMockSession() + { + MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule(); + return (mockAuthenticationModule.createSession(null)); + } } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java index d6e1e681..d0d43329 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java @@ -70,7 +70,7 @@ public class FilesystemBackendModuleTest public void testDeleteFile() throws Exception { QInstance qInstance = TestUtils.defineInstance(); - QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); ///////////////////////////////////////////////////////////////////////////////////////////// // first list the files - then delete one, then re-list, and assert that we have one fewer // @@ -94,7 +94,7 @@ public class FilesystemBackendModuleTest public void testDeleteFileDoesNotExist() throws Exception { QInstance qInstance = TestUtils.defineInstance(); - QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count // @@ -119,7 +119,7 @@ public class FilesystemBackendModuleTest public void testMoveFile() throws Exception { QInstance qInstance = TestUtils.defineInstance(); - QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath(); String subPath = basePath + File.separator + "subdir"; @@ -156,7 +156,7 @@ public class FilesystemBackendModuleTest public void testMoveFileDoesNotExit() throws Exception { QInstance qInstance = TestUtils.defineInstance(); - QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath(); String subPath = basePath + File.separator + "subdir"; List filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java new file mode 100644 index 00000000..183bd1d1 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java @@ -0,0 +1,143 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic; + + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; +import com.kingsrook.qqq.backend.core.callbacks.NoopCallback; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for BasicETLCleanupSourceFilesFunction + *******************************************************************************/ +public class BasicETLCleanupSourceFilesFunctionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDelete() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath = getRandomFilePathPersonTable(qInstance); + + RunFunctionResult runFunctionResult = runFunction(qInstance, filePath, Map.of( + BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, "delete", + BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, "/tmp/trash")); + + assertNull(runFunctionResult.getError()); + assertFalse(new File(filePath).exists(), "File should have been deleted."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMove() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath = getRandomFilePathPersonTable(qInstance); + + String trashDir = File.separator + "tmp" + File.separator + "trash"; + RunFunctionResult runFunctionResult = runFunction(qInstance, filePath, Map.of( + BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, "move", + BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir)); + + assertNull(runFunctionResult.getError()); + assertFalse(new File(filePath).exists(), "File should have been moved."); + + String movedPath = trashDir + File.separator + (new File(filePath).getName()); + assertTrue(new File(movedPath).exists(), "File should have been moved."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private RunFunctionResult runFunction(QInstance qInstance, String filePath, Map values) throws Exception + { + QFunctionMetaData qFunctionMetaData = new BasicETLCleanupSourceFilesFunction().defineFunctionMetaData(); + QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addFunction(qFunctionMetaData); + qInstance.addProcess(qProcessMetaData); + + File file = new File(filePath); + FileUtils.writeStringToFile(file, "content"); + + List records = List.of(new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)); + + RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); + runFunctionRequest.setFunctionName(qFunctionMetaData.getName()); + runFunctionRequest.setProcessName(qProcessMetaData.getName()); + runFunctionRequest.setCallback(new NoopCallback()); + runFunctionRequest.setRecords(records); + runFunctionRequest.setSession(TestUtils.getMockSession()); + runFunctionRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + runFunctionRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3); + + for(Map.Entry entry : values.entrySet()) + { + runFunctionRequest.addValue(entry.getKey(), entry.getValue()); + } + + RunFunctionAction runFunctionAction = new RunFunctionAction(); + return (runFunctionAction.execute(runFunctionRequest)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getRandomFilePathPersonTable(QInstance qInstance) + { + FilesystemBackendMetaData backend = (FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS); + FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS).getBackendDetails(); + String tablePath = backend.getBasePath() + File.separator + backendDetails.getPath(); + String filePath = tablePath + File.separator + UUID.randomUUID(); + return filePath; + } + +} \ No newline at end of file From 1e376b9359c7e2c3c34d14680f02091d19de1507 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 28 Jun 2022 11:28:09 -0500 Subject: [PATCH 26/30] QQQ-14 add FileSyncProcess; Do globbing in s3; Refactoring module vs. base action --- pom.xml | 2 +- .../FilesystemBackendModuleInterface.java | 22 +- .../actions/AbstractBaseFilesystemAction.java | 148 +++++++++- ...AbstractFilesystemTableBackendDetails.java | 72 +++-- .../base/model/metadata/Cardinality.java | 33 +++ .../base/model/metadata/RecordFormat.java | 34 +++ .../local/FilesystemBackendModule.java | 84 +----- .../actions/AbstractFilesystemAction.java | 114 +++++++- .../BasicETLCleanupSourceFilesFunction.java | 40 ++- ...asicETLCollectSourceFileNamesFunction.java | 84 ++++++ .../sync/FilesystemSyncFunction.java | 123 ++++++++ .../sync/FilesystemSyncProcess.java | 78 +++++ .../module/filesystem/s3/S3BackendModule.java | 81 +----- .../s3/actions/AbstractS3Action.java | 132 ++++++++- .../s3/model/metadata/S3BackendMetaData.java | 126 ++++++++- .../module/filesystem/s3/utils/S3Utils.java | 113 ++++++-- .../backend/module/filesystem/TestUtils.java | 20 +- .../local/FilesystemBackendModuleTest.java | 13 +- .../local/actions/FilesystemActionTest.java | 33 ++- .../actions/FilesystemQueryActionTest.java | 26 +- .../FilesystemBackendMetaDataTest.java | 2 +- ...asicETLCleanupSourceFilesFunctionTest.java | 145 ++++++++-- .../sync/FilesystemSyncProcessS3Test.java | 266 ++++++++++++++++++ .../sync/FilesystemSyncProcessTest.java | 119 ++++++++ .../module/filesystem/s3/BaseS3Test.java | 1 + .../s3/S3BackendModuleSubclassForTest.java | 53 ++++ .../filesystem/s3/S3BackendModuleTest.java | 49 ++-- .../model/metadata/S3BackendMetaDataTest.java | 4 +- .../s3/utils/PathMatcherGlobTest.java | 73 +++++ .../filesystem/s3/utils/S3UtilsTest.java | 37 ++- 30 files changed, 1769 insertions(+), 358 deletions(-) create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/Cardinality.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/RecordFormat.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java create mode 100644 src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java create mode 100644 src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java create mode 100644 src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java create mode 100644 src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleSubclassForTest.java create mode 100644 src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/PathMatcherGlobTest.java diff --git a/pom.xml b/pom.xml index 2d069357..0c467092 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ com.kingsrook.qqq qqq-backend-core - 0.0.0-20220623.214704-10 + 0.0.0-20220628.161829-14 diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java index 9a7578a4..3051e18a 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java @@ -22,30 +22,18 @@ package com.kingsrook.qqq.backend.module.filesystem.base; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; -import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; /******************************************************************************* ** Interface to add additional functionality commonly among the various filesystem ** module implementations. *******************************************************************************/ -public interface FilesystemBackendModuleInterface +public interface FilesystemBackendModuleInterface { /******************************************************************************* - ** In contrast with the DeleteAction, which deletes RECORDS - this is a - ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE - ** e.g., for post-ETL. - ** - ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + ** For filesystem backends, get the module-specific action base-class, that helps + ** with functions like listing and deleting files. *******************************************************************************/ - void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException; - - /******************************************************************************* - ** Move a file from a source path, to a destination path. - ** - ** @throws FilesystemException if the move is known to have failed - *******************************************************************************/ - void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException; + AbstractBaseFilesystemAction getActionBase(); } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index b8e07308..1df77c93 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -27,27 +27,42 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import java.util.function.Function; import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractQTableRequest; import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.NotImplementedException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* ** Base class for all Filesystem actions across all modules. + ** + ** @param FILE The class that represents a file in the sub-module. e.g., + * a java.io.File, or an S3Object. *******************************************************************************/ public abstract class AbstractBaseFilesystemAction { + private static final Logger LOG = LogManager.getLogger(AbstractBaseFilesystemAction.class); + + /******************************************************************************* ** List the files for a table - to be implemented in module-specific subclasses. @@ -60,31 +75,79 @@ public abstract class AbstractBaseFilesystemAction public abstract InputStream readFile(FILE file) throws IOException; /******************************************************************************* - ** Add backend details to records about the file that they are in. + ** Write a file - to be implemented in module-specific subclasses. *******************************************************************************/ - protected abstract void addBackendDetailsToRecords(List recordsInFile, FILE file); + public abstract void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException; + + /******************************************************************************* + ** Get a string that represents the full path to a file. + *******************************************************************************/ + protected abstract String getFullPathForFile(FILE file); + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + *******************************************************************************/ + public abstract void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException; + + /******************************************************************************* + ** Move a file from a source path, to a destination path. + ** + ** @throws FilesystemException if the move is known to have failed + *******************************************************************************/ + public abstract void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException; + + /******************************************************************************* + ** e.g., with a base path of /foo/ + ** and a table path of /bar/ + ** and a file at /foo/bar/baz.txt + ** give us just the baz.txt part. + *******************************************************************************/ + public abstract String stripBackendAndTableBasePathsFromFileName(FILE file, QBackendMetaData sourceBackend, QTableMetaData sourceTable); + /******************************************************************************* - ** Append together the backend's base path (if present), with a table's path (again, if present). + ** Append together the backend's base path (if present), with a table's base + ** path (again, if present). *******************************************************************************/ - protected String getFullPath(QTableMetaData table, QBackendMetaData backendBase) + public String getFullBasePath(QTableMetaData table, QBackendMetaData backendBase) { AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase); String fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : ""; AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); - if(StringUtils.hasContent(tableDetails.getPath())) + if(StringUtils.hasContent(tableDetails.getBasePath())) { - fullPath += File.separatorChar + tableDetails.getPath(); + fullPath += File.separatorChar + tableDetails.getBasePath(); } fullPath += File.separatorChar; + fullPath = stripDuplicatedSlashes(fullPath); + return fullPath; } + /******************************************************************************* + ** + *******************************************************************************/ + public static String stripDuplicatedSlashes(String path) + { + if(path == null) + { + return (null); + } + + return (path.replaceAll("//+", "/")); + } + + + /******************************************************************************* ** Get the backend metaData, type-checked as the requested type. *******************************************************************************/ @@ -119,6 +182,8 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ public QueryResult executeQuery(QueryRequest queryRequest) throws QException { + preAction(queryRequest); + try { QueryResult rs = new QueryResult(); @@ -131,20 +196,25 @@ public abstract class AbstractBaseFilesystemAction for(FILE file : files) { + LOG.info("Processing file: " + getFullPathForFile(file)); switch(tableDetails.getRecordFormat()) { - case "csv": + case CSV: { - String fileContents = IOUtils.toString(readFile(file)); + String fileContents = IOUtils.toString(readFile(file)); + fileContents = customizeFileContentsAfterReading(table, fileContents); + List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); addBackendDetailsToRecords(recordsInFile, file); records.addAll(recordsInFile); break; } - case "json": + case JSON: { - String fileContents = IOUtils.toString(readFile(file)); + String fileContents = IOUtils.toString(readFile(file)); + fileContents = customizeFileContentsAfterReading(table, fileContents); + List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); addBackendDetailsToRecords(recordsInFile, file); @@ -162,8 +232,64 @@ public abstract class AbstractBaseFilesystemAction } catch(Exception e) { - e.printStackTrace(); + LOG.warn("Error executing query", e); throw new QException("Error executing query", e); } } + + + + /******************************************************************************* + ** Add backend details to records about the file that they are in. + *******************************************************************************/ + protected void addBackendDetailsToRecords(List recordsInFile, FILE file) + { + recordsInFile.forEach(record -> + { + record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, getFullPathForFile(file)); + }); + } + + + + /******************************************************************************* + ** Method that subclasses can override to add pre-action things (e.g., setting up + ** s3 client). + *******************************************************************************/ + protected void preAction(AbstractQTableRequest tableRequest) + { + ///////////////////////////////////////////////////////////////////// + // noop in base class - subclasses can add functionality if needed // + ///////////////////////////////////////////////////////////////////// + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String customizeFileContentsAfterReading(QTableMetaData table, String fileContents) throws QException + { + Optional optionalCustomizer = table.getCustomizer("postFileRead"); + if(optionalCustomizer.isEmpty()) + { + return (fileContents); + } + QCodeReference customizer = optionalCustomizer.get(); + + try + { + Class customizerClass = Class.forName(customizer.getName()); + + @SuppressWarnings("unchecked") + Function function = (Function) customizerClass.getConstructor().newInstance(); + + return function.apply(fileContents); + } + catch(Exception e) + { + throw (new QException("Error customizing file contents", e)); + } + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java index ca5eae58..e98e4fad 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java @@ -30,42 +30,78 @@ import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; *******************************************************************************/ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails { - private String path; - private String recordFormat; // todo - enum? but hard w/ serialization? - private String cardinality; // todo - enum? + private String basePath; + private String glob; + private RecordFormat recordFormat; + private Cardinality cardinality; /******************************************************************************* - ** Getter for path + ** Getter for basePath ** *******************************************************************************/ - public String getPath() + public String getBasePath() { - return path; + return basePath; } /******************************************************************************* - ** Setter for path + ** Setter for basePath ** *******************************************************************************/ - public void setPath(String path) + public void setBasePath(String basePath) { - this.path = path; + this.basePath = basePath; } /******************************************************************************* - ** Fluent Setter for path + ** Fluent Setter for basePath ** *******************************************************************************/ @SuppressWarnings("unchecked") - public T withPath(String path) + public T withBasePath(String basePath) { - this.path = path; + this.basePath = basePath; + return (T) this; + } + + + + /******************************************************************************* + ** Getter for glob + ** + *******************************************************************************/ + public String getGlob() + { + return glob; + } + + + + /******************************************************************************* + ** Setter for glob + ** + *******************************************************************************/ + public void setGlob(String glob) + { + this.glob = glob; + } + + + + /******************************************************************************* + ** Fluent Setter for glob + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withGlob(String glob) + { + this.glob = glob; return (T) this; } @@ -75,7 +111,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails ** Getter for recordFormat ** *******************************************************************************/ - public String getRecordFormat() + public RecordFormat getRecordFormat() { return recordFormat; } @@ -86,7 +122,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails ** Setter for recordFormat ** *******************************************************************************/ - public void setRecordFormat(String recordFormat) + public void setRecordFormat(RecordFormat recordFormat) { this.recordFormat = recordFormat; } @@ -98,7 +134,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails ** *******************************************************************************/ @SuppressWarnings("unchecked") - public T withRecordFormat(String recordFormat) + public T withRecordFormat(RecordFormat recordFormat) { this.recordFormat = recordFormat; return ((T) this); @@ -110,7 +146,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails ** Getter for cardinality ** *******************************************************************************/ - public String getCardinality() + public Cardinality getCardinality() { return cardinality; } @@ -121,7 +157,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails ** Setter for cardinality ** *******************************************************************************/ - public void setCardinality(String cardinality) + public void setCardinality(Cardinality cardinality) { this.cardinality = cardinality; } @@ -133,7 +169,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails ** *******************************************************************************/ @SuppressWarnings("unchecked") - public T withCardinality(String cardinality) + public T withCardinality(Cardinality cardinality) { this.cardinality = cardinality; return ((T) this); diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/Cardinality.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/Cardinality.java new file mode 100644 index 00000000..0fae2ae6 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/Cardinality.java @@ -0,0 +1,33 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; + + +/******************************************************************************* + ** Mode for filesystem backends: are all records in a single file, or are there + ** many files? + *******************************************************************************/ +public enum Cardinality +{ + ONE, + MANY +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/RecordFormat.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/RecordFormat.java new file mode 100644 index 00000000..572876de --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/RecordFormat.java @@ -0,0 +1,34 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; + + +/******************************************************************************* + ** How are records stored in the files in a filesystem backend? CSV, JSON, + ** future types may XML, or more exotic ones, like "BINARY" or "TEXT" (e.g., 1 + ** record and 1 field per-file) + *******************************************************************************/ +public enum RecordFormat +{ + CSV, + JSON +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java index 09594b99..f432ec7b 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java @@ -24,16 +24,15 @@ package com.kingsrook.qqq.backend.module.filesystem.local; import java.io.File; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; -import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction; @@ -52,6 +51,19 @@ public class FilesystemBackendModule implements QBackendModuleInterface, Filesys private static final Logger LOG = LogManager.getLogger(FilesystemBackendModule.class); + + /******************************************************************************* + ** For filesystem backends, get the module-specific action base-class, that helps + ** with functions like listing and deleting files. + *******************************************************************************/ + @Override + public AbstractBaseFilesystemAction getActionBase() + { + return (new AbstractFilesystemAction()); + } + + + /******************************************************************************* ** Method where a backend module must be able to provide its type (name). *******************************************************************************/ @@ -127,70 +139,4 @@ public class FilesystemBackendModule implements QBackendModuleInterface, Filesys return (new FilesystemDeleteAction()); } - - - /******************************************************************************* - ** In contrast with the DeleteAction, which deletes RECORDS - this is a - ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE - ** e.g., for post-ETL. - ** - ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit - *******************************************************************************/ - @Override - public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException - { - File file = new File(fileReference); - if(!file.exists()) - { - ////////////////////////////////////////////////////////////////////////////////////////////// - // if the file doesn't exist, just exit with noop. don't throw an error - that should only // - // happen if the "contract" of the method is broken, and the file still exists // - ////////////////////////////////////////////////////////////////////////////////////////////// - LOG.debug("Not deleting file [{}], because it does not exist.", file); - return; - } - - if(!file.delete()) - { - throw (new FilesystemException("Failed to delete file: " + fileReference)); - } - } - - - - /******************************************************************************* - ** Move a file from a source path, to a destination path. - ** - ** @throws FilesystemException if the delete is known to have failed - *******************************************************************************/ - @Override - public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException - { - File sourceFile = new File(source); - File destinationFile = new File(destination); - File destinationParent = destinationFile.getParentFile(); - - if(!sourceFile.exists()) - { - throw (new FilesystemException("Cannot move file " + source + ", as it does not exist.")); - } - - ////////////////////////////////////////////////////////////////////////////////////// - // if the destination folder doesn't exist, try to make it - and fail if that fails // - ////////////////////////////////////////////////////////////////////////////////////// - if(!destinationParent.exists()) - { - LOG.debug("Making destination directory {} for move", destinationParent.getAbsolutePath()); - if(!destinationParent.mkdirs()) - { - throw (new FilesystemException("Failed to make destination directory " + destinationParent.getAbsolutePath() + " to move " + source + " into.")); - } - } - - if(!sourceFile.renameTo(destinationFile)) - { - throw (new FilesystemException("Failed to move (rename) file " + source + " to " + destination)); - } - } - } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index 91b5654b..74ddadc2 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -29,11 +29,14 @@ import java.io.InputStream; import java.util.Arrays; import java.util.Collections; import java.util.List; -import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; -import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -41,6 +44,9 @@ import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFile *******************************************************************************/ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction { + private static final Logger LOG = LogManager.getLogger(AbstractFilesystemAction.class); + + /******************************************************************************* ** List the files for this table. @@ -48,7 +54,8 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction @Override public List listFiles(QTableMetaData table, QBackendMetaData backendBase) { - String fullPath = getFullPath(table, backendBase); + // todo - needs rewritten to do globbing... + String fullPath = getFullBasePath(table, backendBase); File directory = new File(fullPath); File[] files = directory.listFiles(); @@ -74,15 +81,106 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction /******************************************************************************* - ** Add backend details to records about the file that they are in. + ** Write a file - to be implemented in module-specific subclasses. *******************************************************************************/ @Override - protected void addBackendDetailsToRecords(List recordsInFile, File file) + public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException { - recordsInFile.forEach(record -> + FileUtils.writeByteArrayToFile(new File(path), contents); + } + + + + /******************************************************************************* + ** Get a string that represents the full path to a file. + *******************************************************************************/ + @Override + protected String getFullPathForFile(File file) + { + return (file.getAbsolutePath()); + } + + + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + *******************************************************************************/ + @Override + public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + { + File file = new File(fileReference); + if(!file.exists()) { - record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, file.getAbsolutePath()); - }); + ////////////////////////////////////////////////////////////////////////////////////////////// + // if the file doesn't exist, just exit with noop. don't throw an error - that should only // + // happen if the "contract" of the method is broken, and the file still exists // + ////////////////////////////////////////////////////////////////////////////////////////////// + LOG.debug("Not deleting file [{}], because it does not exist.", file); + return; + } + + if(!file.delete()) + { + throw (new FilesystemException("Failed to delete file: " + fileReference)); + } + } + + + + /******************************************************************************* + ** Move a file from a source path, to a destination path. + ** + ** @param destination assumed to be a file path - not a directory + ** @throws FilesystemException if the delete is known to have failed + *******************************************************************************/ + @Override + public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException + { + File sourceFile = new File(source); + File destinationFile = new File(destination); + File destinationParent = destinationFile.getParentFile(); + + if(!sourceFile.exists()) + { + throw (new FilesystemException("Cannot move file " + source + ", as it does not exist.")); + } + + ////////////////////////////////////////////////////////////////////////////////////// + // if the destination folder doesn't exist, try to make it - and fail if that fails // + ////////////////////////////////////////////////////////////////////////////////////// + if(!destinationParent.exists()) + { + LOG.debug("Making destination directory {} for move", destinationParent.getAbsolutePath()); + if(!destinationParent.mkdirs()) + { + throw (new FilesystemException("Failed to make destination directory " + destinationParent.getAbsolutePath() + " to move " + source + " into.")); + } + } + + if(!sourceFile.renameTo(destinationFile)) + { + throw (new FilesystemException("Failed to move (rename) file " + source + " to " + destination)); + } + } + + + + /******************************************************************************* + ** e.g., with a base path of /foo/ + ** and a table path of /bar/ + ** and a file at /foo/bar/baz.txt + ** give us just the baz.txt part. + *******************************************************************************/ + @Override + public String stripBackendAndTableBasePathsFromFileName(File file, QBackendMetaData backend, QTableMetaData table) + { + String tablePath = getFullBasePath(table, backend); + String strippedPath = file.getAbsolutePath().replaceFirst(".*" + tablePath, ""); + return (strippedPath); } } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java index e35ad59b..4e096635 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java @@ -23,8 +23,6 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et import java.io.File; -import java.util.Set; -import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; @@ -43,7 +41,8 @@ import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; -import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -52,11 +51,20 @@ import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendD *******************************************************************************/ public class BasicETLCleanupSourceFilesFunction implements FunctionBody { + private static final Logger LOG = LogManager.getLogger(BasicETLCleanupSourceFilesFunction.class); + public static final String FIELD_MOVE_OR_DELETE = "moveOrDelete"; public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves"; + public static final String VALUE_MOVE = "move"; + public static final String VALUE_DELETE = "delete"; + public static final String FUNCTION_NAME = "cleanupSourceFiles"; + + /******************************************************************************* + ** Execute the function - using the request as input, and the result as output. + *******************************************************************************/ @Override public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException { @@ -70,17 +78,22 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function.")); } - Set sourceFiles = runFunctionRequest.getRecords().stream() - .map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH)) - .collect(Collectors.toSet()); + String sourceFilePaths = runFunctionRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS); + if(!StringUtils.hasContent(sourceFilePaths)) + { + LOG.info("No source file paths were specified in field [" + BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS + "]"); + return; + } + + String[] sourceFiles = sourceFilePaths.split(","); for(String sourceFile : sourceFiles) { String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE); - if("delete".equals(moveOrDelete)) + if(VALUE_DELETE.equals(moveOrDelete)) { - filesystemModule.deleteFile(runFunctionRequest.getInstance(), table, sourceFile); + filesystemModule.getActionBase().deleteFile(runFunctionRequest.getInstance(), table, sourceFile); } - else if("move".equals(moveOrDelete)) + else if(VALUE_MOVE.equals(moveOrDelete)) { String destinationForMoves = runFunctionRequest.getValueString(FIELD_DESTINATION_FOR_MOVES); if(!StringUtils.hasContent(destinationForMoves)) @@ -89,11 +102,12 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody } File file = new File(sourceFile); String destinationPath = destinationForMoves + File.separator + file.getName(); - filesystemModule.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath); + filesystemModule.getActionBase().moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath); } else { - throw (new QException("Unexpected value [" + moveOrDelete + "] for field [" + FIELD_MOVE_OR_DELETE + "]. Must be either [move] or [delete].")); + throw (new QException("Unexpected value [" + moveOrDelete + "] for field [" + FIELD_MOVE_OR_DELETE + "]. " + + "Must be either [" + VALUE_MOVE + "] or [" + VALUE_DELETE + "].")); } } } @@ -101,12 +115,12 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody /******************************************************************************* - ** + ** define the metaData that describes this function *******************************************************************************/ public QFunctionMetaData defineFunctionMetaData() { return (new QFunctionMetaData() - .withName("cleanupSourceFiles") + .withName(FUNCTION_NAME) .withCode(new QCodeReference() .withName(this.getClass().getName()) .withCodeType(QCodeType.JAVA) diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java new file mode 100644 index 00000000..20de530f --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunction.java @@ -0,0 +1,84 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic; + + +import java.util.Set; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; + + +/******************************************************************************* + ** Function body for collecting the file names that were discovered in the + ** Extract step. These will be lost during the transform, so we capture them here, + ** so that our Clean function can move or delete them. + ** + ** TODO - need unit test!! + *******************************************************************************/ +public class BasicETLCollectSourceFileNamesFunction implements FunctionBody +{ + public static final String FUNCTION_NAME = "collectSourceFileNames"; + public static final String FIELD_SOURCE_FILE_PATHS = "sourceFilePaths"; + + + + /******************************************************************************* + ** Execute the function - using the request as input, and the result as output. + *******************************************************************************/ + @Override + public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + { + Set sourceFiles = runFunctionRequest.getRecords().stream() + .map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH)) + .collect(Collectors.toSet()); + runFunctionResult.addValue(FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles)); + } + + + + /******************************************************************************* + ** define the metaData that describes this function + *******************************************************************************/ + public QFunctionMetaData defineFunctionMetaData() + { + return (new QFunctionMetaData() + .withName(FUNCTION_NAME) + .withCode(new QCodeReference() + .withName(this.getClass().getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.FUNCTION)) + .withOutputMetaData(new QFunctionOutputMetaData() + .addField(new QFieldMetaData(FIELD_SOURCE_FILE_PATHS, QFieldType.STRING)))); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java new file mode 100644 index 00000000..be043bc7 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java @@ -0,0 +1,123 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync; + + +import java.io.File; +import java.io.InputStream; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Function body for collecting the file names that were discovered in the + ** Extract step. These will be lost during the transform, so we capture them here, + ** so that our Clean function can move or delete them. + ** + *******************************************************************************/ +public class FilesystemSyncFunction implements FunctionBody +{ + private static final Logger LOG = LogManager.getLogger(FilesystemSyncFunction.class); + + public static final String FUNCTION_NAME = "sync"; + + + + /******************************************************************************* + ** Execute the function - using the request as input, and the result as output. + *******************************************************************************/ + @Override + public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + { + QTableMetaData sourceTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE)); + QTableMetaData archiveTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE)); + QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE)); + + QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName()); + FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend); + Map sourceFiles = getFileNames(sourceModule.getActionBase(), sourceTable, sourceBackend); + + QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName()); + FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend); + Set archiveFiles = getFileNames(archiveModule.getActionBase(), archiveTable, archiveBackend).keySet(); + + QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName()); + FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend); + + for(Map.Entry sourceEntry : sourceFiles.entrySet()) + { + try + { + String sourceFileName = sourceEntry.getKey(); + if(!archiveFiles.contains(sourceFileName)) + { + LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]"); + InputStream inputStream = sourceModule.getActionBase().readFile(sourceEntry.getValue()); + byte[] bytes = inputStream.readAllBytes(); + + String archivePath = archiveModule.getActionBase().getFullBasePath(archiveTable, archiveBackend); + archiveModule.getActionBase().writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes); + + String processingPath = processingModule.getActionBase().getFullBasePath(processingTable, processingBackend); + processingModule.getActionBase().writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes); + } + } + catch(Exception e) + { + LOG.error("Error processing file: " + sourceEntry, e); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private Map getFileNames(AbstractBaseFilesystemAction actionBase, QTableMetaData table, QBackendMetaData backend) + { + List files = actionBase.listFiles(table, backend); + Map rs = new LinkedHashMap<>(); + + for(Object file : files) + { + String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(file, backend, table); + rs.put(fileName, file); + } + + return (rs); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java new file mode 100644 index 00000000..37313fb2 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java @@ -0,0 +1,78 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync; + + +import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; + + +/******************************************************************************* + ** Definition for Filesystem sync process. + ** + ** Job is to: + ** - list all files in the source table. + ** - list all files in the archive table. + ** - if any files exist in the source, but not in the archive, then: + ** - copy the file to both the archive and the processing table. + ** + ** The idea being, that the source is read-only, and we want to move files out of + ** processing after they've been processed - and the archive is what we can have + ** in-between the two. + *******************************************************************************/ +public class FilesystemSyncProcess +{ + public static final String PROCESS_NAME = "filesystem.sync"; + + public static final String FIELD_SOURCE_TABLE = "sourceTable"; + public static final String FIELD_ARCHIVE_TABLE = "archiveTable"; + public static final String FIELD_PROCESSING_TABLE = "processingTable"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QProcessMetaData defineProcessMetaData() + { + QFunctionMetaData syncFunction = new QFunctionMetaData() + .withName(FilesystemSyncFunction.FUNCTION_NAME) + .withCode(new QCodeReference() + .withName(FilesystemSyncFunction.class.getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.FUNCTION)) + .withInputData(new QFunctionInputMetaData() + .addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)) + .addField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING)) + .addField(new QFieldMetaData(FIELD_PROCESSING_TABLE, QFieldType.STRING))); + + return new QProcessMetaData() + .withName(PROCESS_NAME) + .addFunction(syncFunction); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java index 33bffb0b..38a54535 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java @@ -22,24 +22,23 @@ package com.kingsrook.qqq.backend.module.filesystem.s3; +import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; -import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; -import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; /******************************************************************************* @@ -47,7 +46,17 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; *******************************************************************************/ public class S3BackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface { - private S3Utils s3Utils; + + + /******************************************************************************* + ** For filesystem backends, get the module-specific action base-class, that helps + ** with functions like listing and deleting files. + *******************************************************************************/ + @Override + public AbstractBaseFilesystemAction getActionBase() + { + return (new AbstractS3Action()); + } @@ -84,43 +93,6 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke - /******************************************************************************* - ** In contrast with the DeleteAction, which deletes RECORDS - this is a - ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE - ** e.g., for post-ETL. - ** - ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit - *******************************************************************************/ - @Override - public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException - { - QBackendMetaData backend = instance.getBackend(table.getBackendName()); - String bucketName = ((S3BackendMetaData) backend).getBucketName(); - - getS3Utils().deleteObject(bucketName, fileReference); - } - - - - - /******************************************************************************* - ** In contrast with the DeleteAction, which deletes RECORDS - this is a - ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE - ** e.g., for post-ETL. - ** - ** @throws FilesystemException if the move is known to have failed - *******************************************************************************/ - @Override - public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException - { - QBackendMetaData backend = instance.getBackend(table.getBackendName()); - String bucketName = ((S3BackendMetaData) backend).getBucketName(); - - getS3Utils().moveObject(bucketName, source, destination); - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -163,29 +135,4 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke return (new S3DeleteAction()); } - - - /******************************************************************************* - ** Setter for s3Utils - *******************************************************************************/ - public void setS3Utils(S3Utils s3Utils) - { - this.s3Utils = s3Utils; - } - - - - /******************************************************************************* - ** Internal accessor for the s3Utils object - should always use this, not the field. - *******************************************************************************/ - private S3Utils getS3Utils() - { - if(s3Utils == null) - { - s3Utils = new S3Utils(); - } - - return s3Utils; - } - } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 1fb11d34..281d3583 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -25,14 +25,22 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; import java.io.IOException; import java.io.InputStream; import java.util.List; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.actions.AbstractQTableRequest; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; -import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -40,10 +48,46 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; *******************************************************************************/ public class AbstractS3Action extends AbstractBaseFilesystemAction { + private static final Logger LOG = LogManager.getLogger(AbstractS3Action.class); + private S3Utils s3Utils; + /******************************************************************************* + ** Setup the s3 utils object to be used for this action. + *******************************************************************************/ + @Override + protected void preAction(AbstractQTableRequest tableRequest) + { + super.preAction(tableRequest); + + if(s3Utils != null) + { + LOG.debug("s3Utils object is already set - not re-setting it."); + return; + } + + S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, tableRequest.getBackend()); + AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard(); + + if(StringUtils.hasContent(s3BackendMetaData.getAccessKey()) && StringUtils.hasContent(s3BackendMetaData.getSecretKey())) + { + BasicAWSCredentials credentials = new BasicAWSCredentials(s3BackendMetaData.getAccessKey(), s3BackendMetaData.getSecretKey()); + clientBuilder.setCredentials(new AWSStaticCredentialsProvider(credentials)); + } + + if(StringUtils.hasContent(s3BackendMetaData.getRegion())) + { + clientBuilder.setRegion(s3BackendMetaData.getRegion()); + } + + s3Utils = new S3Utils(); + s3Utils.setAmazonS3(clientBuilder.build()); + } + + + /******************************************************************************* ** Set the S3Utils object. *******************************************************************************/ @@ -75,17 +119,17 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction listFiles(QTableMetaData table, QBackendMetaData backendBase) { - S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase); + S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase); + AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); - String fullPath = getFullPath(table, backendBase); + String fullPath = getFullBasePath(table, backendBase); String bucketName = s3BackendMetaData.getBucketName(); + String glob = tableDetails.getGlob(); //////////////////////////////////////////////////////////////////// - // todo - read metadata to decide if we should include subfolders // // todo - look at metadata to configure the s3 client here? // //////////////////////////////////////////////////////////////////// - boolean includeSubfolders = false; - return getS3Utils().listObjectsInBucketAtPath(bucketName, fullPath, includeSubfolders); + return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob); } @@ -102,15 +146,77 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction recordsInFile, S3ObjectSummary s3ObjectSummary) + public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException { - recordsInFile.forEach(record -> - { - record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, s3ObjectSummary.getKey()); - }); + path = stripDuplicatedSlashes(path); + String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName(); + getS3Utils().writeFile(bucketName, path, contents); + } + + + /******************************************************************************* + ** Get a string that represents the full path to a file. + *******************************************************************************/ + @Override + protected String getFullPathForFile(S3ObjectSummary s3ObjectSummary) + { + return (s3ObjectSummary.getKey()); + } + + + + /******************************************************************************* + ** e.g., with a base path of /foo/ + ** and a table path of /bar/ + ** and a file at /foo/bar/baz.txt + ** give us just the baz.txt part. + *******************************************************************************/ + @Override + public String stripBackendAndTableBasePathsFromFileName(S3ObjectSummary file, QBackendMetaData backend, QTableMetaData table) + { + String tablePath = getFullBasePath(table, backend); + String strippedPath = file.getKey().replaceFirst(".*" + tablePath, ""); + return (strippedPath); + } + + + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit + *******************************************************************************/ + @Override + public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + { + QBackendMetaData backend = instance.getBackend(table.getBackendName()); + String bucketName = ((S3BackendMetaData) backend).getBucketName(); + + getS3Utils().deleteObject(bucketName, fileReference); + } + + + + /******************************************************************************* + ** In contrast with the DeleteAction, which deletes RECORDS - this is a + ** filesystem-(or s3, sftp, etc)-specific extension to delete an entire FILE + ** e.g., for post-ETL. + ** + ** @param destination assumed to be a file path - not a directory + ** @throws FilesystemException if the move is known to have failed + *******************************************************************************/ + @Override + public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException + { + QBackendMetaData backend = instance.getBackend(table.getBackendName()); + String bucketName = ((S3BackendMetaData) backend).getBucketName(); + + getS3Utils().moveObject(bucketName, source, destination); } } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java index 70bbe0ff..5f064b39 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java @@ -22,16 +22,20 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata; +import com.kingsrook.qqq.backend.core.model.metadata.QSecretReader; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; /******************************************************************************* - ** (local) Filesystem backend meta data. + ** S3 backend meta data. *******************************************************************************/ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData { private String bucketName; + private String accessKey; + private String secretKey; + private String region; @@ -79,4 +83,124 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData return (T) this; } + + + /******************************************************************************* + ** Getter for accessKey + ** + *******************************************************************************/ + public String getAccessKey() + { + return accessKey; + } + + + + /******************************************************************************* + ** Setter for accessKey + ** + *******************************************************************************/ + public void setAccessKey(String accessKey) + { + this.accessKey = accessKey; + } + + + + /******************************************************************************* + ** Fluent setter for accessKey + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withAccessKey(String accessKey) + { + this.accessKey = accessKey; + return (T) this; + } + + + + /******************************************************************************* + ** Getter for secretKey + ** + *******************************************************************************/ + public String getSecretKey() + { + return secretKey; + } + + + + /******************************************************************************* + ** Setter for secretKey + ** + *******************************************************************************/ + public void setSecretKey(String secretKey) + { + this.secretKey = secretKey; + } + + + + /******************************************************************************* + ** Fluent setter for secretKey + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withSecretKey(String secretKey) + { + this.secretKey = secretKey; + return (T) this; + } + + + + /******************************************************************************* + ** Getter for region + ** + *******************************************************************************/ + public String getRegion() + { + return region; + } + + + + /******************************************************************************* + ** Setter for region + ** + *******************************************************************************/ + public void setRegion(String region) + { + this.region = region; + } + + + + /******************************************************************************* + ** Fluent setter for region + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withRegion(String region) + { + this.region = region; + return (T) this; + } + + + + /******************************************************************************* + ** Called by the QInstanceEnricher - to do backend-type-specific enrichments. + ** Original use case is: reading secrets into fields (e.g., passwords). + *******************************************************************************/ + @Override + public void enrich() + { + super.enrich(); + QSecretReader secretReader = new QSecretReader(); + accessKey = secretReader.readSecret(accessKey); + secretKey = secretReader.readSecret(secretKey); + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java index 65a25367..adc2b374 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java @@ -22,7 +22,12 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.utils; +import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.net.URI; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.PathMatcher; import java.util.ArrayList; import java.util.List; import com.amazonaws.regions.Regions; @@ -30,8 +35,10 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.ListObjectsV2Request; import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -47,25 +54,57 @@ public class S3Utils { private static final Logger LOG = LogManager.getLogger(S3Utils.class); - private AmazonS3 s3; + private AmazonS3 amazonS3; /******************************************************************************* - ** List the objects in an S3 bucket at a given path + ** List the objects in an S3 bucket matching a glob, per: + ** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String) *******************************************************************************/ - public List listObjectsInBucketAtPath(String bucketName, String fullPath, boolean includeSubfolders) + public List listObjectsInBucketMatchingGlob(String bucketName, String path, String glob) { ////////////////////////////////////////////////////////////////////////////////////////////////// // s3 list requests find nothing if the path starts with a /, so strip away any leading slashes // // also strip away trailing /'s, for consistent known paths. // // also normalize any duplicated /'s to a single /. // ////////////////////////////////////////////////////////////////////////////////////////////////// - fullPath = fullPath.replaceFirst("^/+", "").replaceFirst("/+$", "").replaceAll("//+", "/"); + path = path.replaceFirst("^/+", "").replaceFirst("/+$", "").replaceAll("//+", "/"); + String prefix = path; + + // todo - maybe this is some error - that the user put a * in the path instead of the glob? + if(prefix.indexOf('*') > -1) + { + prefix = prefix.substring(0, prefix.indexOf('*')); + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob. // + // That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. // + // Anyway, add a trailing /* to globs with no wildcards (or just a '*' if it's a request for the root ("")) // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(glob == null) + { + glob = ""; + } + if(!glob.contains("*")) + { + if(glob.equals("")) + { + glob += "*"; + } + else + { + glob += "/*"; + } + } + + String pathMatcherArg = AbstractFilesystemAction.stripDuplicatedSlashes("glob:/" + path + "/" + glob); + PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher(pathMatcherArg); ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request() .withBucketName(bucketName) - .withPrefix(fullPath); + .withPrefix(prefix); ListObjectsV2Result listObjectsV2Result = null; List rs = new ArrayList<>(); @@ -76,7 +115,8 @@ public class S3Utils { listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken()); } - listObjectsV2Result = getS3().listObjectsV2(listObjectsV2Request); + LOG.info("Listing bucket=" + bucketName + ", path=" + path); + listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request); ////////////////////////////////// // put files in the result list // @@ -85,26 +125,29 @@ public class S3Utils { String key = objectSummary.getKey(); - ////////////////// - // skip folders // - ////////////////// + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // it looks like keys in s3 can have duplicated /'s - so normalize those, to create a "sane" result // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + key = key.replaceAll("//+", "/"); + + //////////////////////////////////////////////////////////////////////////////// + // always skip folders // + // this seemed to fire when it was first written, but not in our unit tests - // + // is this a difference with real s3 vs. localstack possibly? // + //////////////////////////////////////////////////////////////////////////////// if(key.endsWith("/")) { LOG.debug("Skipping file [{}] because it is a folder", key); continue; } - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // if we're not supposed to include subfolders, check the path on this file, and only include it if it matches the request // - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(!includeSubfolders) + /////////////////////////////////////////// + // skip files that do not match the glob // + /////////////////////////////////////////// + if(!pathMatcher.matches(Path.of(URI.create("file:///" + key)))) { - String prefix = key.substring(0, key.lastIndexOf("/")); - if(!prefix.equals(fullPath)) - { - LOG.debug("Skipping file [{}] in a sub-folder [{}] != [{}]", key, prefix, fullPath); - continue; - } + LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob); + continue; } rs.add(objectSummary); @@ -122,7 +165,17 @@ public class S3Utils *******************************************************************************/ public InputStream getObjectAsInputStream(S3ObjectSummary s3ObjectSummary) { - return getS3().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()).getObjectContent(); + return getAmazonS3().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()).getObjectContent(); + } + + + + /******************************************************************************* + ** Write a file + *******************************************************************************/ + public void writeFile(String bucket, String key, byte[] contents) + { + getAmazonS3().putObject(bucket, key, new ByteArrayInputStream(contents), new ObjectMetadata()); } @@ -137,7 +190,7 @@ public class S3Utils ////////////////////////////////////////////////////////////////////////////////////////////// try { - getS3().deleteObject(bucketName, key); + getAmazonS3().deleteObject(bucketName, key); } catch(Exception e) { @@ -157,8 +210,8 @@ public class S3Utils ////////////////////////////////////////////////////////////////////////////////////////////// try { - getS3().copyObject(bucketName, source, bucketName, destination); - getS3().deleteObject(bucketName, source); + getAmazonS3().copyObject(bucketName, source, bucketName, destination); + getAmazonS3().deleteObject(bucketName, source); } catch(Exception e) { @@ -171,9 +224,9 @@ public class S3Utils /******************************************************************************* ** Setter for AmazonS3 client object. *******************************************************************************/ - public void setAmazonS3(AmazonS3 s3) + public void setAmazonS3(AmazonS3 amazonS3) { - this.s3 = s3; + this.amazonS3 = amazonS3; } @@ -181,14 +234,14 @@ public class S3Utils /******************************************************************************* ** Getter for AmazonS3 client object. *******************************************************************************/ - public AmazonS3 getS3() + public AmazonS3 getAmazonS3() { - if(s3 == null) + if(amazonS3 == null) { - s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build(); + // TODO - get this (and other props?) from backend meta data + amazonS3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build(); } - return s3; + return amazonS3; } - } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index d94da070..676e3f96 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -33,6 +33,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.mock.MockAuthenticationModule; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; @@ -56,6 +58,10 @@ public class TestUtils /////////////////////////////////////////////////////////////////// public static final String BASE_PATH = "/tmp/filesystem-tests"; + ////////////////////////////////////////////////////////////////////////////// + // Used to make each test method have a unique folder path, more or less... // + // See methods that work with it. // + ////////////////////////////////////////////////////////////////////////////// private static int testInstanceCounter = 0; @@ -110,7 +116,7 @@ public class TestUtils QInstance qInstance = new QInstance(); qInstance.setAuthentication(defineAuthentication()); qInstance.addBackend(defineLocalFilesystemBackend()); - qInstance.addTable(defineLocalFilesystemCSVPersonTable()); + qInstance.addTable(defineLocalFilesystemJSONPersonTable()); qInstance.addBackend(defineS3Backend()); qInstance.addTable(defineS3CSVPersonTable()); @@ -149,7 +155,7 @@ public class TestUtils /******************************************************************************* ** *******************************************************************************/ - public static QTableMetaData defineLocalFilesystemCSVPersonTable() + public static QTableMetaData defineLocalFilesystemJSONPersonTable() { return new QTableMetaData() .withName(TABLE_NAME_PERSON_LOCAL_FS) @@ -164,9 +170,9 @@ public class TestUtils .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) .withField(new QFieldMetaData("email", QFieldType.STRING)) .withBackendDetails(new FilesystemTableBackendDetails() - .withPath("persons") - .withRecordFormat("json") - .withCardinality("many") + .withBasePath("persons") + .withRecordFormat(RecordFormat.JSON) + .withCardinality(Cardinality.MANY) ); } @@ -203,8 +209,8 @@ public class TestUtils .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) .withField(new QFieldMetaData("email", QFieldType.STRING)) .withBackendDetails(new S3TableBackendDetails() - .withRecordFormat("csv") - .withCardinality("many") + .withRecordFormat(RecordFormat.CSV) + .withCardinality(Cardinality.MANY) ); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java index d0d43329..9d172573 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java @@ -78,7 +78,7 @@ public class FilesystemBackendModuleTest List filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); - filesystemBackendModule.deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath()); + filesystemBackendModule.getActionBase().deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath()); List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); Assertions.assertEquals(filesBeforeDelete.size() - 1, filesAfterDelete.size(), @@ -98,12 +98,13 @@ public class FilesystemBackendModuleTest ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count // - // note, we'd like to detect the non-delete, but there's no such info back from aws it appears? // + // note, our contract is that as long as the file doesn't exist after calling delete (e.g., if it wasn't there // + // to begin with, then we're okay, and don't expect an exception // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// List filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); - filesystemBackendModule.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + filesystemBackendModule.getActionBase().deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); Assertions.assertEquals(filesBeforeDelete.size(), filesAfterDelete.size(), @@ -133,7 +134,7 @@ public class FilesystemBackendModuleTest String originalFilePath = filesBeforeMove.get(0).getAbsolutePath(); String movedFilePath = originalFilePath.replace(basePath, subPath); - filesystemBackendModule.moveFile(qInstance, table, originalFilePath, movedFilePath); + filesystemBackendModule.getActionBase().moveFile(qInstance, table, originalFilePath, movedFilePath); List filesAfterMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); Assertions.assertEquals(filesBeforeMove.size() - 1, filesAfterMove.size(), "Should be one fewer file in the listing after moving one."); @@ -141,7 +142,7 @@ public class FilesystemBackendModuleTest ////////////////////////////////////////////////////////////////////////// // move the file back and assert that the count goes back to the before // ////////////////////////////////////////////////////////////////////////// - filesystemBackendModule.moveFile(qInstance, table, movedFilePath, originalFilePath); + filesystemBackendModule.getActionBase().moveFile(qInstance, table, movedFilePath, originalFilePath); List filesAfterMoveBack = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); Assertions.assertEquals(filesBeforeMove.size(), filesAfterMoveBack.size(), "Should be original number of files after moving back"); @@ -166,7 +167,7 @@ public class FilesystemBackendModuleTest Assertions.assertThrows(FilesystemException.class, () -> { FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); - filesystemBackendModule.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, movedFilePath); + filesystemBackendModule.getActionBase().moveFile(qInstance, table, PATH_THAT_WONT_EXIST, movedFilePath); }); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java index 9e9b38a2..755ab477 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java @@ -29,6 +29,8 @@ import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import static org.junit.jupiter.api.Assertions.fail; @@ -40,6 +42,28 @@ import static org.junit.jupiter.api.Assertions.fail; public class FilesystemActionTest { + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + public void beforeEach() throws Exception + { + primeFilesystem(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + public void afterEach() throws Exception + { + cleanFilesystem(); + } + + + /******************************************************************************* ** Set up the file system *******************************************************************************/ @@ -61,14 +85,17 @@ public class FilesystemActionTest + /******************************************************************************* + ** Write some data files into the directory for the filesystem module. + *******************************************************************************/ private void writePersonFiles(File baseDirectory) throws IOException { String fullPath = baseDirectory.getAbsolutePath(); - if (TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details) + if (TestUtils.defineLocalFilesystemJSONPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details) { - if (StringUtils.hasContent(details.getPath())) + if (StringUtils.hasContent(details.getBasePath())) { - fullPath += File.separatorChar + details.getPath(); + fullPath += File.separatorChar + details.getBasePath(); } } fullPath += File.separatorChar; diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java index f624cb5a..de3e21d8 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java @@ -28,9 +28,7 @@ import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -40,28 +38,6 @@ import org.junit.jupiter.api.Test; public class FilesystemQueryActionTest extends FilesystemActionTest { - /******************************************************************************* - ** - *******************************************************************************/ - @BeforeEach - public void beforeEach() throws Exception - { - super.primeFilesystem(); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @AfterEach - public void afterEach() throws Exception - { - super.cleanFilesystem(); - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -85,7 +61,7 @@ public class FilesystemQueryActionTest extends FilesystemActionTest { QueryRequest queryRequest = new QueryRequest(); queryRequest.setInstance(TestUtils.defineInstance()); - queryRequest.setTableName(TestUtils.defineLocalFilesystemCSVPersonTable().getName()); + queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName()); return queryRequest; } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java index e8b24695..7c6515f6 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java @@ -71,7 +71,7 @@ class FilesystemBackendMetaDataTest QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json); assertThat(deserialized).usingRecursiveComparison() - .ignoringFields("hasBeenValidated") + .ignoringFields("hasBeenValidated") // note, this field is @JsonIgnore .isEqualTo(qInstance); } } \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java index 183bd1d1..8da9dbda 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunctionTest.java @@ -23,20 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et import java.io.File; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.UUID; import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; -import com.kingsrook.qqq.backend.core.callbacks.NoopCallback; import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; -import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; -import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import org.apache.commons.io.FileUtils; @@ -56,17 +55,11 @@ public class BasicETLCleanupSourceFilesFunctionTest ** *******************************************************************************/ @Test - public void testDelete() throws Exception + public void testDelete1Record1File() throws Exception { QInstance qInstance = TestUtils.defineInstance(); String filePath = getRandomFilePathPersonTable(qInstance); - - RunFunctionResult runFunctionResult = runFunction(qInstance, filePath, Map.of( - BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, "delete", - BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, "/tmp/trash")); - - assertNull(runFunctionResult.getError()); - assertFalse(new File(filePath).exists(), "File should have been deleted."); + testDelete(qInstance, List.of(filePath)); } @@ -75,21 +68,11 @@ public class BasicETLCleanupSourceFilesFunctionTest ** *******************************************************************************/ @Test - public void testMove() throws Exception + public void testDelete2Records1File() throws Exception { QInstance qInstance = TestUtils.defineInstance(); String filePath = getRandomFilePathPersonTable(qInstance); - - String trashDir = File.separator + "tmp" + File.separator + "trash"; - RunFunctionResult runFunctionResult = runFunction(qInstance, filePath, Map.of( - BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, "move", - BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir)); - - assertNull(runFunctionResult.getError()); - assertFalse(new File(filePath).exists(), "File should have been moved."); - - String movedPath = trashDir + File.separator + (new File(filePath).getName()); - assertTrue(new File(movedPath).exists(), "File should have been moved."); + testDelete(qInstance, List.of(filePath, filePath)); } @@ -97,25 +80,127 @@ public class BasicETLCleanupSourceFilesFunctionTest /******************************************************************************* ** *******************************************************************************/ - private RunFunctionResult runFunction(QInstance qInstance, String filePath, Map values) throws Exception + @Test + public void testDelete2Record2File() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath1 = getRandomFilePathPersonTable(qInstance); + String filePath2 = getRandomFilePathPersonTable(qInstance); + testDelete(qInstance, List.of(filePath1, filePath2)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMove1Record1File() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath = getRandomFilePathPersonTable(qInstance); + testMove(qInstance, List.of(filePath)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMove2Records1File() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath = getRandomFilePathPersonTable(qInstance); + testMove(qInstance, List.of(filePath, filePath)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMove2Record2File() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + String filePath1 = getRandomFilePathPersonTable(qInstance); + String filePath2 = getRandomFilePathPersonTable(qInstance); + testMove(qInstance, List.of(filePath1, filePath2)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void testDelete(QInstance qInstance, List filePaths) throws Exception + { + RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of( + BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_DELETE, + // todo - even though this field isn't needed, since we gave a value of "delete" + // the RunFunctionAction considers any missing input to be an error... + BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, "")); + + assertNull(runFunctionResult.getError()); + for(String filePath : filePaths) + { + assertFalse(new File(filePath).exists(), "File should have been deleted."); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void testMove(QInstance qInstance, List filePaths) throws Exception + { + String trashDir = File.separator + "tmp" + File.separator + "trash"; + RunFunctionResult runFunctionResult = runFunction(qInstance, filePaths, Map.of( + BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesFunction.VALUE_MOVE, + BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir)); + + assertNull(runFunctionResult.getError()); + + for(String filePath : filePaths) + { + assertFalse(new File(filePath).exists(), "File should have been moved."); + + String movedPath = trashDir + File.separator + (new File(filePath).getName()); + assertTrue(new File(movedPath).exists(), "File should have been moved."); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private RunFunctionResult runFunction(QInstance qInstance, List filePaths, Map values) throws Exception { QFunctionMetaData qFunctionMetaData = new BasicETLCleanupSourceFilesFunction().defineFunctionMetaData(); QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addFunction(qFunctionMetaData); qInstance.addProcess(qProcessMetaData); - File file = new File(filePath); - FileUtils.writeStringToFile(file, "content"); + HashSet filePathsSet = new HashSet<>(filePaths); + for(String filePath : filePathsSet) + { + File file = new File(filePath); + FileUtils.writeStringToFile(file, "content"); + } - List records = List.of(new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)); + // List records = filePaths.stream() + // .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList(); RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); runFunctionRequest.setFunctionName(qFunctionMetaData.getName()); runFunctionRequest.setProcessName(qProcessMetaData.getName()); - runFunctionRequest.setCallback(new NoopCallback()); - runFunctionRequest.setRecords(records); + // runFunctionRequest.setRecords(records); runFunctionRequest.setSession(TestUtils.getMockSession()); runFunctionRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS); runFunctionRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3); + runFunctionRequest.addValue(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet)); for(Map.Entry entry : values.entrySet()) { @@ -135,7 +220,7 @@ public class BasicETLCleanupSourceFilesFunctionTest { FilesystemBackendMetaData backend = (FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS); FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS).getBackendDetails(); - String tablePath = backend.getBasePath() + File.separator + backendDetails.getPath(); + String tablePath = backend.getBasePath() + File.separator + backendDetails.getBasePath(); String filePath = tablePath + File.separator + UUID.randomUUID(); return filePath; } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java new file mode 100644 index 00000000..2064617c --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java @@ -0,0 +1,266 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync; + + +import java.util.List; +import java.util.stream.Collectors; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; +import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; +import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModuleSubclassForTest; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for FilesystemSyncProcess using S3 backend + *******************************************************************************/ +class FilesystemSyncProcessS3Test extends BaseS3Test +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws Exception + { + QBackendModuleDispatcher.registerBackendModule(new S3BackendModuleSubclassForTest()); + QInstance qInstance = TestUtils.defineInstance(); + + String sourceBucket = "source-bucket"; + String archiveBucket = "archive-bucket"; + String processingBucket = "processing-bucket"; + + getAmazonS3().createBucket(sourceBucket); + getAmazonS3().createBucket(archiveBucket); + getAmazonS3().createBucket(processingBucket); + + S3BackendMetaData sourceBackend = defineBackend(qInstance, "source", sourceBucket); + S3BackendMetaData archiveBackend = defineBackend(qInstance, "archive", archiveBucket); + S3BackendMetaData processingBackend = defineBackend(qInstance, "processing", processingBucket); + + QTableMetaData sourceTable = defineTable(qInstance, "source", sourceBackend, "source", "*/l3/*.csv"); + QTableMetaData archiveTable = defineTable(qInstance, "archive", archiveBackend, "archive", "*/l3/*.csv"); + QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv"); + + QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); + QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); + qInstance.addProcess(process); + + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); + + /////////////////////////// + // write some test files // + /////////////////////////// + writeTestFile(sourceBackend, sourceTable, "foo/l3/1.csv", "x"); + writeTestFile(sourceBackend, sourceTable, "bar/l3/2.csv", "x"); + writeTestFile(archiveBackend, archiveTable, "foo/l3/1.csv", "x"); + + printTableListing(sourceBackend, sourceTable); + printTableListing(archiveBackend, archiveTable); + printTableListing(processingBackend, processingTable); + + ////////////////////// + // run the function // + ////////////////////// + RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); + runFunctionRequest.setFunctionName(function.getName()); + runFunctionRequest.setProcessName(process.getName()); + runFunctionRequest.setSession(TestUtils.getMockSession()); + + RunFunctionAction runFunctionAction = new RunFunctionAction(); + RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); + System.out.println(runFunctionResult); + + printTableListing(sourceBackend, sourceTable); + printTableListing(archiveBackend, archiveTable); + printTableListing(processingBackend, processingTable); + + assertTableListing(archiveBackend, archiveTable, "root/archive/foo/l3/1.csv", "root/archive/bar/l3/2.csv"); + assertTableListing(processingBackend, processingTable, "root/processing/bar/l3/2.csv"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testArchiveAndProcessingInSameBucket() throws Exception + { + QBackendModuleDispatcher.registerBackendModule(new S3BackendModuleSubclassForTest()); + QInstance qInstance = TestUtils.defineInstance(); + + String vendorBucket = "vendor-bucket"; + String localBucket = "local-bucket"; + + getAmazonS3().createBucket(vendorBucket); + getAmazonS3().createBucket(localBucket); + + S3BackendMetaData vendorBackend = defineBackend(qInstance, "source", vendorBucket); + S3BackendMetaData localBackend = defineBackend(qInstance, "archive", localBucket); + + QTableMetaData sourceTable = defineTable(qInstance, "source", vendorBackend, "source", "*/l3/*.csv"); + QTableMetaData archiveTable = defineTable(qInstance, "archive", localBackend, "archive", "*/l3/*.csv"); + QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv"); + + QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); + QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); + qInstance.addProcess(process); + + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); + + /////////////////////////// + // write some test files // + /////////////////////////// + writeTestFile(vendorBackend, sourceTable, "foo/l3/1.csv", "x"); + writeTestFile(vendorBackend, sourceTable, "bar/l3/2.csv", "x"); + writeTestFile(localBackend, archiveTable, "foo/l3/1.csv", "x"); + + printTableListing(vendorBackend, sourceTable); + printTableListing(localBackend, archiveTable); + printTableListing(localBackend, processingTable); + + ////////////////////// + // run the function // + ////////////////////// + RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); + runFunctionRequest.setFunctionName(function.getName()); + runFunctionRequest.setProcessName(process.getName()); + runFunctionRequest.setSession(TestUtils.getMockSession()); + + RunFunctionAction runFunctionAction = new RunFunctionAction(); + RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); + System.out.println(runFunctionResult); + + printTableListing(vendorBackend, sourceTable); + printTableListing(localBackend, archiveTable); + printTableListing(localBackend, processingTable); + + assertTableListing(localBackend, archiveTable, "root/archive/foo/l3/1.csv", "root/archive/bar/l3/2.csv"); + assertTableListing(localBackend, processingTable, "root/processing/bar/l3/2.csv"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException + { + S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend); + AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase(); + + List s3ObjectSummaries = actionBase.listFiles(table, backend); + assertEquals(paths.length, s3ObjectSummaries.size(), "Expected number of files in table: " + table.getName()); + for(String path : paths) + { + assertTrue(s3ObjectSummaries.stream().anyMatch(s3o -> s3o.getKey().equals(path)), + "Path [" + path + "] should be in the listing, but was not. Full listing is: " + + s3ObjectSummaries.stream().map(S3ObjectSummary::getKey).collect(Collectors.joining(","))); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException + { + S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend); + AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase(); + + System.out.println("Files in: " + table.getName()); + actionBase.listFiles(table, backend).forEach(o -> System.out.println(o.getKey())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeTestFile(S3BackendMetaData backend, QTableMetaData table, String name, String content) throws Exception + { + S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend); + AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase(); + String fullPath = actionBase.getFullBasePath(table, backend); + + actionBase.writeFile(backend, fullPath + "/" + name, content.getBytes()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private S3BackendMetaData defineBackend(QInstance qInstance, String which, String bucketName) + { + QBackendMetaData backendMetaData = new S3BackendMetaData() + .withBucketName(bucketName) + .withBasePath("root") + .withName("backend-" + which); + qInstance.addBackend(backendMetaData); + return (S3BackendMetaData) backendMetaData; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineTable(QInstance qInstance, String which, QBackendMetaData backend, String path, String glob) + { + QTableMetaData qTableMetaData = new QTableMetaData() + .withName("table-" + which) + .withBackendName(backend.getName()) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withBackendDetails(new S3TableBackendDetails() + .withBasePath(path) + .withGlob(glob)); + qInstance.addTable(qTableMetaData); + return (qTableMetaData); + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java new file mode 100644 index 00000000..73254757 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java @@ -0,0 +1,119 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync; + + +import java.io.File; +import java.io.IOException; +import com.kingsrook.qqq.backend.core.actions.RunFunctionAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for FilesystemSyncProcess + *******************************************************************************/ +class FilesystemSyncProcessTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws Exception + { + QTableMetaData sourceTable = defineTable("source"); + QTableMetaData archiveTable = defineTable("archive"); + QTableMetaData processingTable = defineTable("processing"); + QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData(); + QFunctionMetaData function = process.getFunction(FilesystemSyncFunction.FUNCTION_NAME); + + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName()); + function.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName()); + + QInstance qInstance = TestUtils.defineInstance(); + qInstance.addTable(sourceTable); + qInstance.addTable(archiveTable); + qInstance.addTable(processingTable); + qInstance.addProcess(process); + + /////////////////////////// + // write some test files // + /////////////////////////// + String basePath = ((FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS)).getBasePath(); + writeTestFile(basePath, sourceTable, "1.txt", "x"); + writeTestFile(basePath, sourceTable, "2.txt", "x"); + writeTestFile(basePath, archiveTable, "2.txt", "x"); + + ////////////////////// + // run the function // + ////////////////////// + RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance); + runFunctionRequest.setFunctionName(function.getName()); + runFunctionRequest.setProcessName(process.getName()); + runFunctionRequest.setSession(TestUtils.getMockSession()); + + RunFunctionAction runFunctionAction = new RunFunctionAction(); + RunFunctionResult runFunctionResult = runFunctionAction.execute(runFunctionRequest); + System.out.println(runFunctionResult); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeTestFile(String basePath, QTableMetaData table, String name, String content) throws IOException + { + String path = ((FilesystemTableBackendDetails) table.getBackendDetails()).getBasePath(); + File file = new File(basePath + "/" + path + "/" + name); + FileUtils.writeStringToFile(file, content); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineTable(String subPath) + { + return new QTableMetaData() + .withName("table-" + subPath) + .withBackendName(TestUtils.BACKEND_NAME_LOCAL_FS) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withBackendDetails(new FilesystemTableBackendDetails() + .withBasePath(subPath)); + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java index 3563d91c..490a10b7 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java @@ -59,6 +59,7 @@ public class BaseS3Test amazonS3.putObject(BUCKET_NAME, "0.csv", getCSVHeader()); amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/1.csv", getCSVData1()); amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2()); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test"); amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3()); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleSubclassForTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleSubclassForTest.java new file mode 100644 index 00000000..84e5189f --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleSubclassForTest.java @@ -0,0 +1,53 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3; + + +import com.amazonaws.services.s3.model.S3ObjectSummary; +import cloud.localstack.awssdkv1.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; +import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; + + +/******************************************************************************* + ** Subclass of the S3Backend module, meant for use in unit tests, if/where we + ** need to make sure we use the localstack version of the S3 client. + *******************************************************************************/ +public class S3BackendModuleSubclassForTest extends S3BackendModule +{ + + /******************************************************************************* + ** Seed the AbstractS3Action with an s3Utils object that has the localstack + ** s3 client in it + *******************************************************************************/ + @Override + public AbstractBaseFilesystemAction getActionBase() + { + AbstractS3Action actionBase = (AbstractS3Action) super.getActionBase(); + S3Utils s3Utils = new S3Utils(); + s3Utils.setAmazonS3(TestUtils.getClientS3()); + actionBase.setS3Utils(s3Utils); + return (actionBase); + } + +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java index e0cb0489..44d1796d 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -54,13 +55,14 @@ public class S3BackendModuleTest extends BaseS3Test ///////////////////////////////////////////////////////////////////////////////////////////// // first list the files - then delete one, then re-list, and assert that we have one fewer // ///////////////////////////////////////////////////////////////////////////////////////////// - List s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + List s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); - S3BackendModule s3BackendModule = new S3BackendModule(); - s3BackendModule.setS3Utils(getS3Utils()); - s3BackendModule.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey()); + S3BackendModule s3BackendModule = new S3BackendModule(); + AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); + actionBase.setS3Utils(getS3Utils()); + actionBase.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey()); - List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(), "Should be one fewer file listed after deleting one."); } @@ -80,13 +82,14 @@ public class S3BackendModuleTest extends BaseS3Test // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count // // note, we'd like to detect the non-delete, but there's no such info back from aws it appears? // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// - List s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + List s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); - S3BackendModule s3BackendModule = new S3BackendModule(); - s3BackendModule.setS3Utils(getS3Utils()); - s3BackendModule.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + S3BackendModule s3BackendModule = new S3BackendModule(); + AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); + actionBase.setS3Utils(getS3Utils()); + actionBase.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); - List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); + List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size(), s3ObjectSummariesAfterDelete.size(), "Should be same number of files after deleting bogus path"); } @@ -107,18 +110,19 @@ public class S3BackendModuleTest extends BaseS3Test // first list the files (non-recursively) - then move one into a sub-folder, then re-list, and // // assert that we have one fewer then list again including sub-folders, and see the changed count // //////////////////////////////////////////////////////////////////////////////////////////////////// - List s3ObjectSummariesBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); - List s3ObjectSummariesInSubFolderBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, subPath, false); - List s3ObjectSummariesRecursiveBeforeMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, true); + List s3ObjectSummariesBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); + List s3ObjectSummariesInSubFolderBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, subPath, ""); + List s3ObjectSummariesRecursiveBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**"); - S3BackendModule s3BackendModule = new S3BackendModule(); - s3BackendModule.setS3Utils(getS3Utils()); + S3BackendModule s3BackendModule = new S3BackendModule(); + AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); + actionBase.setS3Utils(getS3Utils()); String key = s3ObjectSummariesBeforeMove.get(0).getKey(); - s3BackendModule.moveFile(qInstance, table, key, key.replaceFirst(TEST_FOLDER, subPath)); + actionBase.moveFile(qInstance, table, key, key.replaceFirst(TEST_FOLDER, subPath)); - List s3ObjectSummariesAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, false); - List s3ObjectSummariesRecursiveAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER, true); - List s3ObjectSummariesInSubFolderAfterMove = getS3Utils().listObjectsInBucketAtPath(BUCKET_NAME, subPath, false); + List s3ObjectSummariesAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); + List s3ObjectSummariesRecursiveAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**"); + List s3ObjectSummariesInSubFolderAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, subPath, ""); Assertions.assertEquals(s3ObjectSummariesBeforeMove.size() - 1, s3ObjectSummariesAfterMove.size(), "Should be one fewer file in the non-recursive listing after moving one."); @@ -140,11 +144,12 @@ public class S3BackendModuleTest extends BaseS3Test QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3); String subPath = TEST_FOLDER + "/" + SUB_FOLDER; - S3BackendModule s3BackendModule = new S3BackendModule(); - s3BackendModule.setS3Utils(getS3Utils()); + S3BackendModule s3BackendModule = new S3BackendModule(); + AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); + actionBase.setS3Utils(getS3Utils()); Assertions.assertThrows(FilesystemException.class, () -> - s3BackendModule.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, subPath + "/" + UUID.randomUUID()) + actionBase.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, subPath + "/" + UUID.randomUUID()) ); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java index 6211d4ce..acef8ec6 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java @@ -52,7 +52,7 @@ class S3BackendMetaDataTest System.out.println(JsonUtils.prettyPrint(json)); System.out.println(json); String expectToContain = """ - {"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","backendType":"s3","name":"s3"}"""; + {"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","secretKey":null,"accessKey":null,"backendType":"s3","name":"s3","region":null}"""; assertTrue(json.contains(expectToContain)); } @@ -71,7 +71,7 @@ class S3BackendMetaDataTest QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json); assertThat(deserialized).usingRecursiveComparison() - .ignoringFields("hasBeenValidated") + .ignoringFields("hasBeenValidated") // note, this field is @JsonIgnore .isEqualTo(qInstance); } } \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/PathMatcherGlobTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/PathMatcherGlobTest.java new file mode 100644 index 00000000..2aa65293 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/PathMatcherGlobTest.java @@ -0,0 +1,73 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.utils; + + +import java.net.URI; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Verification for some of the behavior in the S3Utils - working with PathMatcher + ** globs. + *******************************************************************************/ +public class PathMatcherGlobTest +{ + + @Test + public void testPathMatcher() throws Exception + { + ///////////////////////////////////////////////////////////// + // note: must start with for both the pattern and the uri // + ///////////////////////////////////////////////////////////// + PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/*/acme/*/*.csv"); + + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/1234.csv"))), "Glob should match"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/nj/acme/20220627/1234.csv"))), "Glob should match"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220628/1234.csv"))), "Glob should match"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/12345.csv"))), "Glob should match"); + + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/beta/20220627/1234.csv"))), "Glob should not match (beta vs acme)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///something/stl/acme/20220627/1234.csv"))), "Glob should not match (wrong start path)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/csv"))), "Glob should not match (no file basename)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/1234.CSV"))), "Glob should not match (wrong case extension)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/extra/1234.csv"))), "Glob should not match (extra dir)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/extra/acme/20220627/1234.csv"))), "Glob should not match (extra dir)"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/extra/stl/acme/20220627/1234.csv"))), "Glob should not match (extra dir)"); + + pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/**/acme/*/*.csv"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/extra/stl/acme/20220627/1234.csv"))), "Glob should match with extra dir"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/extra/extra2/stl/acme/20220627/1234.csv"))), "Glob should match with 2 extra dirs"); + Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/acme/20220627/1234.csv"))), "Glob does not match with no dir for **"); + + pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/**"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/1234.csv"))), "Glob should match with extra dir"); + + pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/*"); + Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///1234.csv"))), "Glob should match"); + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java index 0de6c5e2..ed4afbdf 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.utils; import java.io.IOException; import java.io.InputStream; +import java.util.List; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import org.apache.commons.io.IOUtils; @@ -44,15 +45,23 @@ public class S3UtilsTest extends BaseS3Test public void testListObjectsInBucketAtPath() { S3Utils s3Utils = getS3Utils(); - assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/", false).size(), "Expected # of s3 objects without subfolders"); - assertEquals(3, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/", true).size(), "Expected # of s3 objects with subfolders"); - assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "/" + TEST_FOLDER + "/", false).size(), "With leading slash"); - assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "/" + TEST_FOLDER, false).size(), "Without trailing slash"); - assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "//" + TEST_FOLDER + "//", false).size(), "With multiple leading and trailing slashes"); - assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, false).size(), "Just in the subfolder non-recursive"); - assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, true).size(), "Just in the subfolder recursive"); - assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "//" + SUB_FOLDER, true).size(), "Just in the subfolder recursive"); - assertEquals(0, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "not-a-real-path/", true).size(), "In a non-existing folder"); + assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders"); + assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders"); + assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders"); + assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders"); + assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash"); + assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash"); + assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, "").size(), "Just in the subfolder non-recursive"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, "/**").size(), "Just in the subfolder recursive"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "//" + SUB_FOLDER, "/**").size(), "Just in the subfolder recursive, multi /"); + assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "not-a-real-path/", "").size(), "In a non-existing folder"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s"); + assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string"); + assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively"); + assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively"); } @@ -63,12 +72,12 @@ public class S3UtilsTest extends BaseS3Test @Test public void testGetObjectAsInputStream() throws IOException { - S3Utils s3Utils = getS3Utils(); - S3ObjectSummary s3ObjectSummary = s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "test-files", true).get(0); - InputStream inputStream = s3Utils.getObjectAsInputStream(s3ObjectSummary); - String csvFromS3 = IOUtils.toString(inputStream); + S3Utils s3Utils = getS3Utils(); + List s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", ""); + S3ObjectSummary s3ObjectSummary = s3ObjectSummaries.stream().filter(o -> o.getKey().contains("1.csv")).findAny().get(); + InputStream inputStream = s3Utils.getObjectAsInputStream(s3ObjectSummary); + String csvFromS3 = IOUtils.toString(inputStream); - // todo - should check the filename somewhere, right? assertEquals(getCSVData1(), csvFromS3, "File from S3 should match expected content"); } From 4bc0ab1c4913be7c4d92eccfe85aed959742c725 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 28 Jun 2022 13:49:37 -0500 Subject: [PATCH 27/30] QQQ-14 fixes to sync process --- .../actions/AbstractBaseFilesystemAction.java | 5 ++- .../sync/FilesystemSyncFunction.java | 32 +++++++++++-------- .../s3/actions/AbstractS3Action.java | 24 +++++++++++--- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 1df77c93..e5c69153 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -32,7 +32,6 @@ import java.util.function.Function; import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.AbstractQTableRequest; import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -182,7 +181,7 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ public QueryResult executeQuery(QueryRequest queryRequest) throws QException { - preAction(queryRequest); + preAction(queryRequest.getBackend()); try { @@ -256,7 +255,7 @@ public abstract class AbstractBaseFilesystemAction ** Method that subclasses can override to add pre-action things (e.g., setting up ** s3 client). *******************************************************************************/ - protected void preAction(AbstractQTableRequest tableRequest) + public void preAction(QBackendMetaData backendMetaData) { ///////////////////////////////////////////////////////////////////// // noop in base class - subclasses can add functionality if needed // diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java index be043bc7..5ca6c732 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java @@ -65,16 +65,22 @@ public class FilesystemSyncFunction implements FunctionBody QTableMetaData archiveTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE)); QTableMetaData processingTable = runFunctionRequest.getInstance().getTable(runFunctionRequest.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE)); - QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName()); - FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend); - Map sourceFiles = getFileNames(sourceModule.getActionBase(), sourceTable, sourceBackend); + QBackendMetaData sourceBackend = runFunctionRequest.getInstance().getBackendForTable(sourceTable.getName()); + FilesystemBackendModuleInterface sourceModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend); + AbstractBaseFilesystemAction sourceActionBase = sourceModule.getActionBase(); + sourceActionBase.preAction(sourceBackend); + Map sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend); - QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName()); - FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend); - Set archiveFiles = getFileNames(archiveModule.getActionBase(), archiveTable, archiveBackend).keySet(); + QBackendMetaData archiveBackend = runFunctionRequest.getInstance().getBackendForTable(archiveTable.getName()); + FilesystemBackendModuleInterface archiveModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend); + AbstractBaseFilesystemAction archiveActionBase = archiveModule.getActionBase(); + archiveActionBase.preAction(archiveBackend); + Set archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet(); - QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName()); - FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend); + QBackendMetaData processingBackend = runFunctionRequest.getInstance().getBackendForTable(processingTable.getName()); + FilesystemBackendModuleInterface processingModule = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend); + AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase(); + processingActionBase.preAction(processingBackend); for(Map.Entry sourceEntry : sourceFiles.entrySet()) { @@ -84,14 +90,14 @@ public class FilesystemSyncFunction implements FunctionBody if(!archiveFiles.contains(sourceFileName)) { LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]"); - InputStream inputStream = sourceModule.getActionBase().readFile(sourceEntry.getValue()); + InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue()); byte[] bytes = inputStream.readAllBytes(); - String archivePath = archiveModule.getActionBase().getFullBasePath(archiveTable, archiveBackend); - archiveModule.getActionBase().writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes); + String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend); + archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes); - String processingPath = processingModule.getActionBase().getFullBasePath(processingTable, processingBackend); - processingModule.getActionBase().writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes); + String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend); + processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes); } } catch(Exception e) diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 281d3583..a7ae421b 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -29,7 +29,6 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.kingsrook.qqq.backend.core.model.actions.AbstractQTableRequest; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; @@ -58,9 +57,9 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction Date: Wed, 29 Jun 2022 10:21:19 -0500 Subject: [PATCH 28/30] QQQ-14 checkpoint, pre-demo --- pom.xml | 2 +- .../base/actions/AbstractBaseFilesystemAction.java | 4 ++-- .../local/actions/AbstractFilesystemAction.java | 6 +++--- .../basic/BasicETLCleanupSourceFilesFunction.java | 13 ++++++++----- .../filesystem/sync/FilesystemSyncFunction.java | 11 ++++++++++- .../filesystem/sync/FilesystemSyncProcess.java | 11 ++++++++--- .../filesystem/s3/actions/AbstractS3Action.java | 7 ++++--- .../s3/model/metadata/S3BackendMetaData.java | 9 +++++---- .../backend/module/filesystem/s3/utils/S3Utils.java | 4 +++- .../filesystem/sync/FilesystemSyncProcessTest.java | 4 ++++ 10 files changed, 48 insertions(+), 23 deletions(-) diff --git a/pom.xml b/pom.xml index 0c467092..05300b35 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ com.kingsrook.qqq qqq-backend-core - 0.0.0-20220628.161829-14 + 0.0.0-20220629.151616-15 diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index e5c69153..9a04f289 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -81,7 +81,7 @@ public abstract class AbstractBaseFilesystemAction /******************************************************************************* ** Get a string that represents the full path to a file. *******************************************************************************/ - protected abstract String getFullPathForFile(FILE file); + public abstract String getFullPathForFile(FILE file); /******************************************************************************* ** In contrast with the DeleteAction, which deletes RECORDS - this is a @@ -105,7 +105,7 @@ public abstract class AbstractBaseFilesystemAction ** and a file at /foo/bar/baz.txt ** give us just the baz.txt part. *******************************************************************************/ - public abstract String stripBackendAndTableBasePathsFromFileName(FILE file, QBackendMetaData sourceBackend, QTableMetaData sourceTable); + public abstract String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData sourceBackend, QTableMetaData sourceTable); diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index 74ddadc2..be5dd44b 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -95,7 +95,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction ** Get a string that represents the full path to a file. *******************************************************************************/ @Override - protected String getFullPathForFile(File file) + public String getFullPathForFile(File file) { return (file.getAbsolutePath()); } @@ -176,10 +176,10 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction ** give us just the baz.txt part. *******************************************************************************/ @Override - public String stripBackendAndTableBasePathsFromFileName(File file, QBackendMetaData backend, QTableMetaData table) + public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table) { String tablePath = getFullBasePath(table, backend); - String strippedPath = file.getAbsolutePath().replaceFirst(".*" + tablePath, ""); + String strippedPath = filePath.replaceFirst(".*" + tablePath, ""); return (strippedPath); } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java index 4e096635..f91a5b10 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesFunction.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -56,7 +57,7 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody public static final String FIELD_MOVE_OR_DELETE = "moveOrDelete"; public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves"; - public static final String VALUE_MOVE = "move"; + public static final String VALUE_MOVE = "move"; public static final String VALUE_DELETE = "delete"; public static final String FUNCTION_NAME = "cleanupSourceFiles"; @@ -77,6 +78,8 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody { throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function.")); } + AbstractBaseFilesystemAction actionBase = filesystemModule.getActionBase(); + actionBase.preAction(backend); String sourceFilePaths = runFunctionRequest.getValueString(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS); if(!StringUtils.hasContent(sourceFilePaths)) @@ -91,7 +94,7 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE); if(VALUE_DELETE.equals(moveOrDelete)) { - filesystemModule.getActionBase().deleteFile(runFunctionRequest.getInstance(), table, sourceFile); + actionBase.deleteFile(runFunctionRequest.getInstance(), table, sourceFile); } else if(VALUE_MOVE.equals(moveOrDelete)) { @@ -100,9 +103,9 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody { throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value.")); } - File file = new File(sourceFile); - String destinationPath = destinationForMoves + File.separator + file.getName(); - filesystemModule.getActionBase().moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath); + String filePathWithoutBase = actionBase.stripBackendAndTableBasePathsFromFileName(sourceFile, backend, table); + String destinationPath = destinationForMoves + File.separator + filePathWithoutBase; + actionBase.moveFile(runFunctionRequest.getInstance(), table, sourceFile, destinationPath); } else { diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java index 5ca6c732..45264376 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncFunction.java @@ -82,6 +82,8 @@ public class FilesystemSyncFunction implements FunctionBody AbstractBaseFilesystemAction processingActionBase = processingModule.getActionBase(); processingActionBase.preAction(processingBackend); + Integer maxFilesToSync = runFunctionRequest.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE); + int syncedFileCount = 0; for(Map.Entry sourceEntry : sourceFiles.entrySet()) { try @@ -98,6 +100,13 @@ public class FilesystemSyncFunction implements FunctionBody String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend); processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes); + syncedFileCount++; + + if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync) + { + LOG.info("Breaking after syncing " + syncedFileCount + " files"); + break; + } } } catch(Exception e) @@ -119,7 +128,7 @@ public class FilesystemSyncFunction implements FunctionBody for(Object file : files) { - String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(file, backend, table); + String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table); rs.put(fileName, file); } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java index 37313fb2..5773630a 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java @@ -41,6 +41,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; ** - if any files exist in the source, but not in the archive, then: ** - copy the file to both the archive and the processing table. ** + ** The maxFilesToArchive field can be used to only sync up to that many files + ** (an help an initial sync, if you want to do it in smaller batches) + ** ** The idea being, that the source is read-only, and we want to move files out of ** processing after they've been processed - and the archive is what we can have ** in-between the two. @@ -49,9 +52,10 @@ public class FilesystemSyncProcess { public static final String PROCESS_NAME = "filesystem.sync"; - public static final String FIELD_SOURCE_TABLE = "sourceTable"; - public static final String FIELD_ARCHIVE_TABLE = "archiveTable"; - public static final String FIELD_PROCESSING_TABLE = "processingTable"; + public static final String FIELD_SOURCE_TABLE = "sourceTable"; + public static final String FIELD_ARCHIVE_TABLE = "archiveTable"; + public static final String FIELD_PROCESSING_TABLE = "processingTable"; + public static final String FIELD_MAX_FILES_TO_ARCHIVE = "maxFilesToArchive"; @@ -69,6 +73,7 @@ public class FilesystemSyncProcess .withInputData(new QFunctionInputMetaData() .addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)) .addField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING)) + .addField(new QFieldMetaData(FIELD_MAX_FILES_TO_ARCHIVE, QFieldType.INTEGER).withDefaultValue(Integer.MAX_VALUE)) .addField(new QFieldMetaData(FIELD_PROCESSING_TABLE, QFieldType.STRING))); return new QProcessMetaData() diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index a7ae421b..21958ab7 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -175,7 +175,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction Date: Fri, 1 Jul 2022 11:24:07 -0500 Subject: [PATCH 29/30] Update qqq-backend-core to 0.0.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 05300b35..3619aa46 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ com.kingsrook.qqq qqq-backend-core - 0.0.0-20220629.151616-15 + 0.0.0 From e084f8f711bbe89cd334e3f9138b37eef467e154 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 1 Jul 2022 11:24:24 -0500 Subject: [PATCH 30/30] Update versions for release --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3619aa46..83edccf9 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ com.kingsrook.qqq qqq-backend-module-filesystem - 0.0.0-SNAPSHOT + 0.0.0 scm:git:git@github.com:Kingsrook/qqq-backend-module-rdbms.git