diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java new file mode 100644 index 00000000..07b5f3dc --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -0,0 +1,163 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.actions; + + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; +import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** Base class for all Filesystem actions across all modules. + *******************************************************************************/ +public abstract class AbstractBaseFilesystemAction +{ + + /******************************************************************************* + ** List the files for a table - to be implemented in module-specific subclasses. + *******************************************************************************/ + public abstract List listFiles(QTableMetaData table, QBackendMetaData backendBase); + + /******************************************************************************* + ** Read the contents of a file - to be implemented in module-specific subclasses. + *******************************************************************************/ + public abstract InputStream readFile(FILE file) throws IOException; + + + + /******************************************************************************* + ** Append together the backend's base path (if present), with a table's path (again, if present). + *******************************************************************************/ + protected String getFullPath(QTableMetaData table, QBackendMetaData backendBase) + { + AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase); + String fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : ""; + + AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); + if(StringUtils.hasContent(tableDetails.getPath())) + { + fullPath += File.separatorChar + tableDetails.getPath(); + } + + fullPath += File.separatorChar; + return fullPath; + } + + + + /******************************************************************************* + ** Get the backend metaData, type-checked as the requested type. + *******************************************************************************/ + protected T getBackendMetaData(Class outputClass, QBackendMetaData metaData) + { + if(!(outputClass.isInstance(metaData))) + { + throw new IllegalArgumentException("MetaData was not of expected type (was " + metaData.getClass().getSimpleName() + ")"); + } + return outputClass.cast(metaData); + } + + + + /******************************************************************************* + ** Get the backendDetails out of a table, type-checked as the requested type + *******************************************************************************/ + protected T getTableBackendDetails(Class outputClass, QTableMetaData tableMetaData) + { + QTableBackendDetails tableBackendDetails = tableMetaData.getBackendDetails(); + if(!(outputClass.isInstance(tableBackendDetails))) + { + throw new IllegalArgumentException("Table backend details was not of expected type (was " + tableBackendDetails.getClass().getSimpleName() + ")"); + } + return outputClass.cast(tableBackendDetails); + } + + + + /******************************************************************************* + ** Generic implementation of the execute method from the QueryInterface + *******************************************************************************/ + public QueryResult executeQuery(QueryRequest queryRequest) throws QException + { + try + { + QueryResult rs = new QueryResult(); + List records = new ArrayList<>(); + rs.setRecords(records); + + QTableMetaData table = queryRequest.getTable(); + AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); + List files = listFiles(table, queryRequest.getBackend()); + + for(FILE file : files) + { + switch(tableDetails.getRecordFormat()) + { + case "csv": + { + String fileContents = IOUtils.toString(readFile(file)); + List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); + + records.addAll(recordsInFile); + break; + } + case "json": + { + String fileContents = IOUtils.toString(readFile(file)); + List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); + + records.addAll(recordsInFile); + break; + } + default: + { + throw new NotImplementedException("Filesystem record format " + tableDetails.getRecordFormat() + " is not yet implemented"); + } + } + } + + return rs; + } + catch(Exception e) + { + e.printStackTrace(); + throw new QException("Error executing query", e); + } + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java new file mode 100644 index 00000000..9002bdb8 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java @@ -0,0 +1,80 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; + + +/******************************************************************************* + ** Base class for all BackendMetaData for all filesystem-style backend modules. + *******************************************************************************/ +public class AbstractFilesystemBackendMetaData extends QBackendMetaData +{ + private String basePath; + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public AbstractFilesystemBackendMetaData() + { + super(); + } + + + + /******************************************************************************* + ** Getter for basePath + ** + *******************************************************************************/ + public String getBasePath() + { + return (basePath); + } + + + + /******************************************************************************* + ** Setter for basePath + ** + *******************************************************************************/ + public void setBasePath(String basePath) + { + this.basePath = basePath; + } + + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withBasePath(String basePath) + { + this.basePath = basePath; + return (T) this; + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java new file mode 100644 index 00000000..ca5eae58 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java @@ -0,0 +1,142 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; + + +/******************************************************************************* + ** Extension of QTableBackendDetails, with details specific to a Filesystem table. + *******************************************************************************/ +public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails +{ + private String path; + private String recordFormat; // todo - enum? but hard w/ serialization? + private String cardinality; // todo - enum? + + + + /******************************************************************************* + ** Getter for path + ** + *******************************************************************************/ + public String getPath() + { + return path; + } + + + + /******************************************************************************* + ** Setter for path + ** + *******************************************************************************/ + public void setPath(String path) + { + this.path = path; + } + + + + /******************************************************************************* + ** Fluent Setter for path + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withPath(String path) + { + this.path = path; + return (T) this; + } + + + + /******************************************************************************* + ** Getter for recordFormat + ** + *******************************************************************************/ + public String getRecordFormat() + { + return recordFormat; + } + + + + /******************************************************************************* + ** Setter for recordFormat + ** + *******************************************************************************/ + public void setRecordFormat(String recordFormat) + { + this.recordFormat = recordFormat; + } + + + + /******************************************************************************* + ** Fluent Setter for recordFormat + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withRecordFormat(String recordFormat) + { + this.recordFormat = recordFormat; + return ((T) this); + } + + + + /******************************************************************************* + ** Getter for cardinality + ** + *******************************************************************************/ + public String getCardinality() + { + return cardinality; + } + + + + /******************************************************************************* + ** Setter for cardinality + ** + *******************************************************************************/ + public void setCardinality(String cardinality) + { + this.cardinality = cardinality; + } + + + + /******************************************************************************* + ** Fluent Setter for cardinality + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withCardinality(String cardinality) + { + this.cardinality = cardinality; + return ((T) this); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java new file mode 100644 index 00000000..25a9e8c7 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java @@ -0,0 +1,119 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package com.kingsrook.qqq.backend.module.filesystem.local; + + +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemUpdateAction; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; + + +/******************************************************************************* + ** QQQ Backend module for working with (local) Filesystems. + *******************************************************************************/ +public class FilesystemBackendModule implements QBackendModuleInterface +{ + + + /******************************************************************************* + ** Method where a backend module must be able to provide its type (name). + *******************************************************************************/ + @Override + public String getBackendType() + { + return ("filesystem"); + } + + + + /******************************************************************************* + ** Method to identify the class used for backend meta data for this module. + *******************************************************************************/ + @Override + public Class getBackendMetaDataClass() + { + return (FilesystemBackendMetaData.class); + } + + + /******************************************************************************* + ** Method to identify the class used for table-backend details for this module. + *******************************************************************************/ + @Override + public Class getTableBackendDetailsClass() + { + return FilesystemTableBackendDetails.class; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryInterface getQueryInterface() + { + return new FilesystemQueryAction(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InsertInterface getInsertInterface() + { + return (new FilesystemInsertAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public UpdateInterface getUpdateInterface() + { + return (new FilesystemUpdateAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public DeleteInterface getDeleteInterface() + { + return (new FilesystemDeleteAction()); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java new file mode 100644 index 00000000..fde5c1d5 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -0,0 +1,64 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; + + +/******************************************************************************* + ** Base class for all (local) Filesystem actions + *******************************************************************************/ +public class AbstractFilesystemAction extends AbstractBaseFilesystemAction +{ + + /******************************************************************************* + ** List the files for this table. + *******************************************************************************/ + @Override + public List listFiles(QTableMetaData table, QBackendMetaData backendBase) + { + String fullPath = getFullPath(table, backendBase); + File directory = new File(fullPath); + return Arrays.asList(directory.listFiles()); + } + + + + /******************************************************************************* + ** Read the contents of a file. + *******************************************************************************/ + @Override + public InputStream readFile(File file) throws IOException + { + return (new FileInputStream(file)); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java new file mode 100644 index 00000000..4d6d9f92 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java @@ -0,0 +1,59 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest; +import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilesystemDeleteAction implements DeleteInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public DeleteResult execute(DeleteRequest deleteRequest) throws QException + { + try + { + DeleteResult rs = new DeleteResult(); + QTableMetaData table = deleteRequest.getTable(); + + throw new NotImplementedException("Filesystem delete not implemented"); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing delete: " + e.getMessage(), e); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java new file mode 100644 index 00000000..328d7b43 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java @@ -0,0 +1,65 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest; +import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult; +import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilesystemInsertAction implements InsertInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public InsertResult execute(InsertRequest insertRequest) throws QException + { + try + { + InsertResult rs = new InsertResult(); + QTableMetaData table = insertRequest.getTable(); + + List recordsWithStatus = new ArrayList<>(); + rs.setRecords(recordsWithStatus); + + throw new NotImplementedException("Filesystem insert not implemented"); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing insert: " + e.getMessage(), e); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryAction.java new file mode 100644 index 00000000..56dc6d5d --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; +import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilesystemQueryAction extends AbstractFilesystemAction implements QueryInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public QueryResult execute(QueryRequest queryRequest) throws QException + { + return executeQuery(queryRequest); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java new file mode 100644 index 00000000..bae14d1b --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java @@ -0,0 +1,65 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest; +import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult; +import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilesystemUpdateAction implements UpdateInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public UpdateResult execute(UpdateRequest updateRequest) throws QException + { + try + { + UpdateResult rs = new UpdateResult(); + QTableMetaData table = updateRequest.getTable(); + + List recordsWithStatus = new ArrayList<>(); + rs.setRecords(recordsWithStatus); + + throw new NotImplementedException("Filesystem update not implemented"); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing update: " + e.getMessage(), e); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java new file mode 100644 index 00000000..02b3950e --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule; + + +/******************************************************************************* + ** (local) Filesystem backend meta data. + *******************************************************************************/ +public class FilesystemBackendMetaData extends AbstractFilesystemBackendMetaData +{ + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public FilesystemBackendMetaData() + { + super(); + setBackendType(FilesystemBackendModule.class); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemTableBackendDetails.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemTableBackendDetails.java new file mode 100644 index 00000000..b2f8cee9 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemTableBackendDetails.java @@ -0,0 +1,44 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule; + + +/******************************************************************************* + ** (local) Filesystem specific Extension of QTableBackendDetails + *******************************************************************************/ +public class FilesystemTableBackendDetails extends AbstractFilesystemTableBackendDetails +{ + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public FilesystemTableBackendDetails() + { + super(); + setBackendType(FilesystemBackendModule.class); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java new file mode 100644 index 00000000..7c574cf3 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java @@ -0,0 +1,121 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3; + + +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; + + +/******************************************************************************* + ** QQQ Backend module for working with AWS S3 filesystems + *******************************************************************************/ +public class S3BackendModule implements QBackendModuleInterface +{ + + + /******************************************************************************* + ** Method where a backend module must be able to provide its type (name). + *******************************************************************************/ + @Override + public String getBackendType() + { + return ("s3"); + } + + + + /******************************************************************************* + ** Method to identify the class used for backend meta data for this module. + *******************************************************************************/ + @Override + public Class getBackendMetaDataClass() + { + return (S3BackendMetaData.class); + } + + + + /******************************************************************************* + ** Method to identify the class used for table-backend details for this module. + *******************************************************************************/ + @Override + public Class getTableBackendDetailsClass() + { + return S3TableBackendDetails.class; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryInterface getQueryInterface() + { + return new S3QueryAction(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InsertInterface getInsertInterface() + { + return (new S3InsertAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public UpdateInterface getUpdateInterface() + { + return (new S3UpdateAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public DeleteInterface getDeleteInterface() + { + return (new S3DeleteAction()); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java new file mode 100644 index 00000000..12c69959 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -0,0 +1,99 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; + + +/******************************************************************************* + ** Base class for all S3 filesystem actions + *******************************************************************************/ +public class AbstractS3Action extends AbstractBaseFilesystemAction +{ + private S3Utils s3Utils; + + + + /******************************************************************************* + ** Set the S3Utils object. + *******************************************************************************/ + public void setS3Utils(S3Utils s3Utils) + { + this.s3Utils = s3Utils; + } + + + + /******************************************************************************* + ** Internal accessor for the s3Utils object - should always use this, not the field. + *******************************************************************************/ + private S3Utils getS3Utils() + { + if(s3Utils == null) + { + s3Utils = new S3Utils(); + } + + return s3Utils; + } + + + + /******************************************************************************* + ** List the files for a table. + *******************************************************************************/ + @Override + public List listFiles(QTableMetaData table, QBackendMetaData backendBase) + { + S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase); + + String fullPath = getFullPath(table, backendBase); + String bucketName = s3BackendMetaData.getBucketName(); + + //////////////////////////////////////////////////////////////////// + // todo - read metadata to decide if we should include subfolders // + // todo - look at metadata to configure the s3 client here? // + //////////////////////////////////////////////////////////////////// + boolean includeSubfolders = false; + return getS3Utils().listObjectsInBucketAtPath(bucketName, fullPath, includeSubfolders); + } + + + + /******************************************************************************* + ** Read the contents of a file. + *******************************************************************************/ + @Override + public InputStream readFile(S3ObjectSummary s3ObjectSummary) throws IOException + { + return (getS3Utils().getObjectAsInputStream(s3ObjectSummary)); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java new file mode 100644 index 00000000..8598c10e --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java @@ -0,0 +1,59 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest; +import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3DeleteAction implements DeleteInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public DeleteResult execute(DeleteRequest deleteRequest) throws QException + { + try + { + DeleteResult rs = new DeleteResult(); + QTableMetaData table = deleteRequest.getTable(); + + throw new NotImplementedException("S3 delete not implemented"); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing delete: " + e.getMessage(), e); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java new file mode 100644 index 00000000..310709b1 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java @@ -0,0 +1,65 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest; +import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult; +import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3InsertAction implements InsertInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public InsertResult execute(InsertRequest insertRequest) throws QException + { + try + { + InsertResult rs = new InsertResult(); + QTableMetaData table = insertRequest.getTable(); + + List recordsWithStatus = new ArrayList<>(); + rs.setRecords(recordsWithStatus); + + throw new NotImplementedException("S3 insert not implemented"); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing insert: " + e.getMessage(), e); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryAction.java new file mode 100644 index 00000000..46e71e25 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; +import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3QueryAction extends AbstractS3Action implements QueryInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public QueryResult execute(QueryRequest queryRequest) throws QException + { + return (super.executeQuery(queryRequest)); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java new file mode 100644 index 00000000..31844f47 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java @@ -0,0 +1,65 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest; +import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult; +import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3UpdateAction implements UpdateInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public UpdateResult execute(UpdateRequest updateRequest) throws QException + { + try + { + UpdateResult rs = new UpdateResult(); + QTableMetaData table = updateRequest.getTable(); + + List recordsWithStatus = new ArrayList<>(); + rs.setRecords(recordsWithStatus); + + throw new NotImplementedException("S3 update not implemented"); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing update: " + e.getMessage(), e); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java new file mode 100644 index 00000000..70bbe0ff --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java @@ -0,0 +1,82 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; + + +/******************************************************************************* + ** (local) Filesystem backend meta data. + *******************************************************************************/ +public class S3BackendMetaData extends AbstractFilesystemBackendMetaData +{ + private String bucketName; + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public S3BackendMetaData() + { + super(); + setBackendType(S3BackendModule.class); + } + + + + /******************************************************************************* + ** Getter for bucketName + ** + *******************************************************************************/ + public String getBucketName() + { + return bucketName; + } + + + + /******************************************************************************* + ** Setter for bucketName + ** + *******************************************************************************/ + public void setBucketName(String bucketName) + { + this.bucketName = bucketName; + } + + + + /******************************************************************************* + ** Fluent setter for bucketName + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public T withBucketName(String bucketName) + { + this.bucketName = bucketName; + return (T) this; + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3TableBackendDetails.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3TableBackendDetails.java new file mode 100644 index 00000000..ba372cc6 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3TableBackendDetails.java @@ -0,0 +1,46 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; + + +/******************************************************************************* + ** S3 specific Extension of QTableBackendDetails + *******************************************************************************/ +public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails +{ + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public S3TableBackendDetails() + { + super(); + setBackendType(S3BackendModule.class); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java new file mode 100644 index 00000000..4dfd4f1c --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java @@ -0,0 +1,147 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.utils; + + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Utility methods for working with AWS S3. + *******************************************************************************/ +public class S3Utils +{ + private static final Logger LOG = LogManager.getLogger(S3Utils.class); + + private AmazonS3 s3; + + + + /******************************************************************************* + ** List the objects in an S3 bucket at a given path + *******************************************************************************/ + public List listObjectsInBucketAtPath(String bucketName, String fullPath, boolean includeSubfolders) + { + ////////////////////////////////////////////////////////////////////////////////////////////////// + // s3 list requests find nothing if the path starts with a /, so strip away any leading slashes // + // also strip away trailing /'s, for consistent known paths. // + // also normalize any duplicated /'s to a single /. // + ////////////////////////////////////////////////////////////////////////////////////////////////// + fullPath = fullPath.replaceFirst("^/+", "").replaceFirst("/+$", "").replaceAll("//+", "/"); + + ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request() + .withBucketName(bucketName) + .withPrefix(fullPath); + + ListObjectsV2Result listObjectsV2Result = null; + List rs = new ArrayList<>(); + + do + { + if(listObjectsV2Result != null) + { + listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken()); + } + listObjectsV2Result = getS3().listObjectsV2(listObjectsV2Request); + + ////////////////////////////////// + // put files in the result list // + ////////////////////////////////// + for(S3ObjectSummary objectSummary : listObjectsV2Result.getObjectSummaries()) + { + String key = objectSummary.getKey(); + + ////////////////// + // skip folders // + ////////////////// + if(key.endsWith("/")) + { + LOG.debug("Skipping file [{}] because it is a folder", key); + continue; + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we're not supposed to include subfolders, check the path on this file, and only include it if it matches the request // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(!includeSubfolders) + { + String prefix = key.substring(0, key.lastIndexOf("/")); + if(!prefix.equals(fullPath)) + { + LOG.debug("Skipping file [{}] in a sub-folder [{}] != [{}]", key, prefix, fullPath); + continue; + } + } + + rs.add(objectSummary); + } + } + while(listObjectsV2Result.isTruncated()); + + return rs; + } + + + /******************************************************************************* + ** + *******************************************************************************/ + public InputStream getObjectAsInputStream(S3ObjectSummary s3ObjectSummary) + { + return getS3().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()).getObjectContent(); + } + + + /******************************************************************************* + ** Setter for AmazonS3 client object. + *******************************************************************************/ + public void setAmazonS3(AmazonS3 s3) + { + this.s3 = s3; + } + + + + /******************************************************************************* + ** Getter for AmazonS3 client object. + *******************************************************************************/ + public AmazonS3 getS3() + { + if(s3 == null) + { + s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build(); + } + + return s3; + } + + +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java new file mode 100644 index 00000000..c624b4a7 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -0,0 +1,185 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem; + + +import java.io.File; +import java.io.IOException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import org.apache.commons.io.FileUtils; + + +/******************************************************************************* + ** Utility methods for Filesystem module tests + *******************************************************************************/ +public class TestUtils +{ + private static int testInstanceCounter = 0; + private final static String BASE_PATH = "/tmp/filesystem-tests"; + + + + /******************************************************************************* + ** Meant to be called in a @BeforeEach - increment an internal counter that will + ** give us a unique directory name for each test method. + *******************************************************************************/ + public static void increaseTestInstanceCounter() + { + testInstanceCounter++; + } + + + + /******************************************************************************* + ** Reset the counter to 0 (e.g., to let some tests have a known value). + *******************************************************************************/ + public static void resetTestInstanceCounter() + { + testInstanceCounter = 0; + } + + + + /******************************************************************************* + ** Get the current value of the testInstanceCounter. See {@link #increaseTestInstanceCounter()} + *******************************************************************************/ + public static int getTestInstanceCounter() + { + return (testInstanceCounter); + } + + + + /******************************************************************************* + ** Meant to run both after and before test methods - makes sure the file system + ** is empty for the path under the instance. + *******************************************************************************/ + public static void cleanInstanceFiles() throws IOException + { + FileUtils.deleteDirectory(new File(BASE_PATH + File.separator + testInstanceCounter)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QInstance defineInstance() throws QInstanceValidationException + { + QInstance qInstance = new QInstance(); + qInstance.addBackend(defineLocalFilesystemBackend()); + qInstance.addTable(defineLocalFilesystemCSVPersonTable()); + qInstance.addBackend(defineS3Backend()); + qInstance.addTable(defineS3CSVPersonTable()); + + new QInstanceValidator().validate(qInstance); + + return (qInstance); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static FilesystemBackendMetaData defineLocalFilesystemBackend() + { + return (new FilesystemBackendMetaData() + .withBasePath(BASE_PATH + File.separator + testInstanceCounter) + .withName("local-filesystem")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineLocalFilesystemCSVPersonTable() + { + return new QTableMetaData() + .withName("person") + .withLabel("Person") + .withBackendName(defineLocalFilesystemBackend().getName()) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date")) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) + .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name")) + .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name")) + .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) + .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withBackendDetails(new FilesystemTableBackendDetails() + .withPath("persons") + .withRecordFormat("csv") + .withCardinality("many") + ); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static S3BackendMetaData defineS3Backend() + { + return (new S3BackendMetaData() + .withBucketName(BaseS3Test.BUCKET_NAME) + .withBasePath(BaseS3Test.TEST_FOLDER) + .withName("s3")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineS3CSVPersonTable() + { + return new QTableMetaData() + .withName("person-s3") + .withLabel("Person S3 Table") + .withBackendName(defineS3Backend().getName()) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date")) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) + .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name")) + .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name")) + .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) + .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withBackendDetails(new S3TableBackendDetails() + .withRecordFormat("csv") + .withCardinality("many") + ); + } + +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java new file mode 100644 index 00000000..7d19512e --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java @@ -0,0 +1,103 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import java.io.File; +import java.io.IOException; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import org.apache.commons.io.FileUtils; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** Base class for Filesystem action tests. + ** + ** Knows how to set up the filesystem for the tests. + *******************************************************************************/ +public class FilesystemActionTest +{ + + /******************************************************************************* + ** Set up the file system + *******************************************************************************/ + protected void primeFilesystem() throws IOException + { + TestUtils.cleanInstanceFiles(); + TestUtils.increaseTestInstanceCounter(); + FilesystemBackendMetaData filesystemBackendMetaData = TestUtils.defineLocalFilesystemBackend(); + + File baseDirectory = new File(filesystemBackendMetaData.getBasePath()); + boolean mkdirsResult = baseDirectory.mkdirs(); + if(!mkdirsResult) + { + fail("Failed to make directories at [" + baseDirectory + "] for filesystem backend module"); + } + + writePersonFiles(baseDirectory); + } + + + + private void writePersonFiles(File baseDirectory) throws IOException + { + String fullPath = baseDirectory.getAbsolutePath(); + if (TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details) + { + if (StringUtils.hasContent(details.getPath())) + { + fullPath += File.separatorChar + details.getPath(); + } + } + fullPath += File.separatorChar; + + String csvHeader = """ + "id","createDate","modifyDate","firstName","lastName","birthDate","email" + """; + + String csvData1 = csvHeader + """ + "1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com" + "2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com" + """; + FileUtils.writeStringToFile(new File(fullPath + "DATA-1.csv"), csvData1); + + String csvData2 = csvHeader + """ + "3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com" + "4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com" + "5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com" + """; + FileUtils.writeStringToFile(new File(fullPath + "DATA-2.csv"), csvData2); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void cleanFilesystem() throws IOException + { + TestUtils.cleanInstanceFiles(); + } +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java new file mode 100644 index 00000000..b918e457 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java @@ -0,0 +1,88 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilesystemQueryActionTest extends FilesystemActionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + public void beforeEach() throws Exception + { + super.primeFilesystem(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + public void afterEach() throws Exception + { + super.cleanFilesystem(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQuery1() throws QException + { + QueryRequest queryRequest = initQueryRequest(); + QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest); + Assertions.assertEquals(5, queryResult.getRecords().size(), "Unfiltered query should find all rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QueryRequest initQueryRequest() throws QInstanceValidationException + { + QueryRequest queryRequest = new QueryRequest(); + queryRequest.setInstance(TestUtils.defineInstance()); + queryRequest.setTableName(TestUtils.defineLocalFilesystemCSVPersonTable().getName()); + return queryRequest; + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java new file mode 100644 index 00000000..e8b24695 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata; + + +import java.io.IOException; +import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for FilesystemBackendMetaData + *******************************************************************************/ +class FilesystemBackendMetaDataTest +{ + + + /******************************************************************************* + ** Test that an instance can be serialized as expected + *******************************************************************************/ + @Test + public void testSerializingToJson() throws QInstanceValidationException + { + TestUtils.resetTestInstanceCounter(); + QInstance qInstance = TestUtils.defineInstance(); + String json = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance); + System.out.println(JsonUtils.prettyPrint(json)); + System.out.println(json); + String expectToContain = """ + "local-filesystem":{"basePath":"/tmp/filesystem-tests/0","backendType":"filesystem","name":"local-filesystem"}"""; + assertTrue(json.contains(expectToContain)); + } + + + + /******************************************************************************* + ** Test that an instance can be deserialized as expected + *******************************************************************************/ + @Test + public void testDeserializingFromJson() throws IOException, QInstanceValidationException + { + QInstanceAdapter qInstanceAdapter = new QInstanceAdapter(); + + QInstance qInstance = TestUtils.defineInstance(); + String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance); + + QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json); + assertThat(deserialized).usingRecursiveComparison() + .ignoringFields("hasBeenValidated") + .isEqualTo(qInstance); + } +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java new file mode 100644 index 00000000..3fb38d3a --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java @@ -0,0 +1,173 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3; + + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + + +/******************************************************************************* + ** + *******************************************************************************/ +@Testcontainers +public class BaseS3Test +{ + private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("localstack/localstack"); + + public static final String BUCKET_NAME = "localstack-test-bucket"; + public static final String TEST_FOLDER = "test-files"; + public static final String SUB_FOLDER = "sub-folder"; + + @Container + private static LocalStackContainer localStack = new LocalStackContainer(DEFAULT_IMAGE_NAME) + .withServices(LocalStackContainer.Service.S3); + + + + /******************************************************************************* + ** Before each unit test, get the test bucket into a known state + *******************************************************************************/ + @BeforeEach + public void beforeEach() + { + AmazonS3 amazonS3 = getAmazonS3(); + + amazonS3.createBucket(BUCKET_NAME); + amazonS3.putObject(BUCKET_NAME, "0.csv", getCSVHeader()); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/1.csv", getCSVData1()); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2()); + amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3()); + } + + + + /******************************************************************************* + ** After each unit test, clean up the bucket + *******************************************************************************/ + @AfterEach + public void afterEach() + { + AmazonS3 amazonS3 = getAmazonS3(); + + if(amazonS3.doesBucketExistV2(BUCKET_NAME)) + { + //////////////////////// + // todo - paginate... // + //////////////////////// + for(S3ObjectSummary objectSummary : amazonS3.listObjectsV2(BUCKET_NAME).getObjectSummaries()) + { + amazonS3.deleteObject(BUCKET_NAME, objectSummary.getKey()); + } + amazonS3.deleteBucket(BUCKET_NAME); + } + } + + + + /******************************************************************************* + ** Access a localstack-configured s3 client. + *******************************************************************************/ + protected AmazonS3 getAmazonS3() + { + BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(localStack.getAccessKey(), localStack.getSecretKey()); + AmazonS3 amazonS3 = AmazonS3ClientBuilder.standard() + .withEndpointConfiguration(localStack.getEndpointConfiguration(LocalStackContainer.Service.S3)) + .withCredentials(new AWSStaticCredentialsProvider(basicAWSCredentials)) + .build(); + return (amazonS3); + } + + + + /******************************************************************************* + ** Access the S3Utils object, with localstack-configured s3 client. + *******************************************************************************/ + protected S3Utils getS3Utils() + { + S3Utils s3Utils = new S3Utils(); + s3Utils.setAmazonS3(getAmazonS3()); + return (s3Utils); + } + + + + /******************************************************************************* + ** Access a string of CSV test data. + *******************************************************************************/ + protected String getCSVHeader() + { + return (""" + "id","createDate","modifyDate","firstName","lastName","birthDate","email" + """); + } + + + + /******************************************************************************* + ** Access a string of CSV test data. + *******************************************************************************/ + protected String getCSVData1() + { + return (getCSVHeader() + """ + "1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com" + "2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com" + """); + } + + + + /******************************************************************************* + ** Access a string of CSV test data. + *******************************************************************************/ + protected String getCSVData2() + { + return (getCSVHeader() + """ + "3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com" + "4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com" + "5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com" + """); + } + + + + /******************************************************************************* + ** Access a string of CSV test data. + *******************************************************************************/ + protected String getCSVData3() + { + return (getCSVHeader() + """ + "6","2022-06-20 15:31:02","2022-06-20 15:31:02","Lisa","S","1986-06-06","lisa.s@kingsrook.com" + """); + } + +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java new file mode 100644 index 00000000..6fe9d2e0 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3QueryActionTest extends BaseS3Test +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQuery1() throws QException + { + QueryRequest queryRequest = initQueryRequest(); + S3QueryAction s3QueryAction = new S3QueryAction(); + s3QueryAction.setS3Utils(getS3Utils()); + QueryResult queryResult = s3QueryAction.execute(queryRequest); + Assertions.assertEquals(5, queryResult.getRecords().size(), "Expected # of rows from unfiltered query"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QueryRequest initQueryRequest() throws QInstanceValidationException + { + QueryRequest queryRequest = new QueryRequest(); + queryRequest.setInstance(TestUtils.defineInstance()); + queryRequest.setTableName(TestUtils.defineS3CSVPersonTable().getName()); + return queryRequest; + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java new file mode 100644 index 00000000..6211d4ce --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata; + + +import java.io.IOException; +import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for S3BackendMetaData + *******************************************************************************/ +class S3BackendMetaDataTest +{ + + + /******************************************************************************* + ** Test that an instance can be serialized as expected + *******************************************************************************/ + @Test + public void testSerializingToJson() throws QInstanceValidationException + { + TestUtils.resetTestInstanceCounter(); + QInstance qInstance = TestUtils.defineInstance(); + String json = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance); + System.out.println(JsonUtils.prettyPrint(json)); + System.out.println(json); + String expectToContain = """ + {"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","backendType":"s3","name":"s3"}"""; + assertTrue(json.contains(expectToContain)); + } + + + + /******************************************************************************* + ** Test that an instance can be deserialized as expected + *******************************************************************************/ + @Test + public void testDeserializingFromJson() throws IOException, QInstanceValidationException + { + QInstanceAdapter qInstanceAdapter = new QInstanceAdapter(); + + QInstance qInstance = TestUtils.defineInstance(); + String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance); + + QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json); + assertThat(deserialized).usingRecursiveComparison() + .ignoringFields("hasBeenValidated") + .isEqualTo(qInstance); + } +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java new file mode 100644 index 00000000..0de6c5e2 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java @@ -0,0 +1,75 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.utils; + + +import java.io.IOException; +import java.io.InputStream; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class S3UtilsTest extends BaseS3Test +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testListObjectsInBucketAtPath() + { + S3Utils s3Utils = getS3Utils(); + assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/", false).size(), "Expected # of s3 objects without subfolders"); + assertEquals(3, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/", true).size(), "Expected # of s3 objects with subfolders"); + assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "/" + TEST_FOLDER + "/", false).size(), "With leading slash"); + assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "/" + TEST_FOLDER, false).size(), "Without trailing slash"); + assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "//" + TEST_FOLDER + "//", false).size(), "With multiple leading and trailing slashes"); + assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, false).size(), "Just in the subfolder non-recursive"); + assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, true).size(), "Just in the subfolder recursive"); + assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "//" + SUB_FOLDER, true).size(), "Just in the subfolder recursive"); + assertEquals(0, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "not-a-real-path/", true).size(), "In a non-existing folder"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testGetObjectAsInputStream() throws IOException + { + S3Utils s3Utils = getS3Utils(); + S3ObjectSummary s3ObjectSummary = s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "test-files", true).get(0); + InputStream inputStream = s3Utils.getObjectAsInputStream(s3ObjectSummary); + String csvFromS3 = IOUtils.toString(inputStream); + + // todo - should check the filename somewhere, right? + assertEquals(getCSVData1(), csvFromS3, "File from S3 should match expected content"); + } + +} \ No newline at end of file