QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

QQQ-14 initial checkin

Browse Source
This commit is contained in:
darin.kelkhoff
2022-06-20 16:07:50 -05:00
parent b3696ee558
commit d60111466b
28 changed files with 2465 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java Normal file
View File

@ -0,0 +1,163 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base.actions;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
** Base class for all Filesystem actions across all modules.
*******************************************************************************/
public abstract class AbstractBaseFilesystemAction<FILE>
{
/*******************************************************************************
** List the files for a table - to be implemented in module-specific subclasses.
*******************************************************************************/
public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase);
/*******************************************************************************
** Read the contents of a file - to be implemented in module-specific subclasses.
*******************************************************************************/
public abstract InputStream readFile(FILE file) throws IOException;
/*******************************************************************************
** Append together the backend's base path (if present), with a table's path (again, if present).
*******************************************************************************/
protected String getFullPath(QTableMetaData table, QBackendMetaData backendBase)
{
AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase);
String fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : "";
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
if(StringUtils.hasContent(tableDetails.getPath()))
{
fullPath += File.separatorChar + tableDetails.getPath();
}
fullPath += File.separatorChar;
return fullPath;
}
/*******************************************************************************
** Get the backend metaData, type-checked as the requested type.
*******************************************************************************/
protected <T extends AbstractFilesystemBackendMetaData> T getBackendMetaData(Class<T> outputClass, QBackendMetaData metaData)
{
if(!(outputClass.isInstance(metaData)))
{
throw new IllegalArgumentException("MetaData was not of expected type (was " + metaData.getClass().getSimpleName() + ")");
}
return outputClass.cast(metaData);
}
/*******************************************************************************
** Get the backendDetails out of a table, type-checked as the requested type
*******************************************************************************/
protected <T extends AbstractFilesystemTableBackendDetails> T getTableBackendDetails(Class<T> outputClass, QTableMetaData tableMetaData)
{
QTableBackendDetails tableBackendDetails = tableMetaData.getBackendDetails();
if(!(outputClass.isInstance(tableBackendDetails)))
{
throw new IllegalArgumentException("Table backend details was not of expected type (was " + tableBackendDetails.getClass().getSimpleName() + ")");
}
return outputClass.cast(tableBackendDetails);
}
/*******************************************************************************
** Generic implementation of the execute method from the QueryInterface
*******************************************************************************/
public QueryResult executeQuery(QueryRequest queryRequest) throws QException
{
try
{
QueryResult rs = new QueryResult();
List<QRecord> records = new ArrayList<>();
rs.setRecords(records);
QTableMetaData table = queryRequest.getTable();
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
List<FILE> files = listFiles(table, queryRequest.getBackend());
for(FILE file : files)
{
switch(tableDetails.getRecordFormat())
{
case "csv":
{
String fileContents = IOUtils.toString(readFile(file));
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
records.addAll(recordsInFile);
break;
}
case "json":
{
String fileContents = IOUtils.toString(readFile(file));
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
records.addAll(recordsInFile);
break;
}
default:
{
throw new NotImplementedException("Filesystem record format " + tableDetails.getRecordFormat() + " is not yet implemented");
}
}
}
return rs;
}
catch(Exception e)
{
e.printStackTrace();
throw new QException("Error executing query", e);
}
}
}

80
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java Normal file
View File

@ -0,0 +1,80 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
/*******************************************************************************
** Base class for all BackendMetaData for all filesystem-style backend modules.
*******************************************************************************/
public class AbstractFilesystemBackendMetaData extends QBackendMetaData
{
private String basePath;
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public AbstractFilesystemBackendMetaData()
{
super();
}
/*******************************************************************************
** Getter for basePath
**
*******************************************************************************/
public String getBasePath()
{
return (basePath);
}
/*******************************************************************************
** Setter for basePath
**
*******************************************************************************/
public void setBasePath(String basePath)
{
this.basePath = basePath;
}
/*******************************************************************************
** Fluent setter for basePath
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends AbstractFilesystemBackendMetaData> T withBasePath(String basePath)
{
this.basePath = basePath;
return (T) this;
}
}

142
src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java Normal file
View File

@ -0,0 +1,142 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
/*******************************************************************************
** Extension of QTableBackendDetails, with details specific to a Filesystem table.
*******************************************************************************/
public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
{
private String path;
private String recordFormat; // todo - enum? but hard w/ serialization?
private String cardinality; // todo - enum?
/*******************************************************************************
** Getter for path
**
*******************************************************************************/
public String getPath()
{
return path;
}
/*******************************************************************************
** Setter for path
**
*******************************************************************************/
public void setPath(String path)
{
this.path = path;
}
/*******************************************************************************
** Fluent Setter for path
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends AbstractFilesystemTableBackendDetails> T withPath(String path)
{
this.path = path;
return (T) this;
}
/*******************************************************************************
** Getter for recordFormat
**
*******************************************************************************/
public String getRecordFormat()
{
return recordFormat;
}
/*******************************************************************************
** Setter for recordFormat
**
*******************************************************************************/
public void setRecordFormat(String recordFormat)
{
this.recordFormat = recordFormat;
}
/*******************************************************************************
** Fluent Setter for recordFormat
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends AbstractFilesystemTableBackendDetails> T withRecordFormat(String recordFormat)
{
this.recordFormat = recordFormat;
return ((T) this);
}
/*******************************************************************************
** Getter for cardinality
**
*******************************************************************************/
public String getCardinality()
{
return cardinality;
}
/*******************************************************************************
** Setter for cardinality
**
*******************************************************************************/
public void setCardinality(String cardinality)
{
this.cardinality = cardinality;
}
/*******************************************************************************
** Fluent Setter for cardinality
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends AbstractFilesystemTableBackendDetails> T withCardinality(String cardinality)
{
this.cardinality = cardinality;
return ((T) this);
}
}

119
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java Normal file
View File

@ -0,0 +1,119 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemUpdateAction;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
/*******************************************************************************
** QQQ Backend module for working with (local) Filesystems.
*******************************************************************************/
public class FilesystemBackendModule implements QBackendModuleInterface
{
/*******************************************************************************
** Method where a backend module must be able to provide its type (name).
*******************************************************************************/
@Override
public String getBackendType()
{
return ("filesystem");
}
/*******************************************************************************
** Method to identify the class used for backend meta data for this module.
*******************************************************************************/
@Override
public Class<? extends QBackendMetaData> getBackendMetaDataClass()
{
return (FilesystemBackendMetaData.class);
}
/*******************************************************************************
** Method to identify the class used for table-backend details for this module.
*******************************************************************************/
@Override
public Class<? extends QTableBackendDetails> getTableBackendDetailsClass()
{
return FilesystemTableBackendDetails.class;
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QueryInterface getQueryInterface()
{
return new FilesystemQueryAction();
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public InsertInterface getInsertInterface()
{
return (new FilesystemInsertAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public UpdateInterface getUpdateInterface()
{
return (new FilesystemUpdateAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public DeleteInterface getDeleteInterface()
{
return (new FilesystemDeleteAction());
}
}

64
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java Normal file
View File

@ -0,0 +1,64 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
/*******************************************************************************
** Base class for all (local) Filesystem actions
*******************************************************************************/
public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
{
/*******************************************************************************
** List the files for this table.
*******************************************************************************/
@Override
public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase)
{
String fullPath = getFullPath(table, backendBase);
File directory = new File(fullPath);
return Arrays.asList(directory.listFiles());
}
/*******************************************************************************
** Read the contents of a file.
*******************************************************************************/
@Override
public InputStream readFile(File file) throws IOException
{
return (new FileInputStream(file));
}
}

59
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java Normal file
View File

@ -0,0 +1,59 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class FilesystemDeleteAction implements DeleteInterface
{
/*******************************************************************************
**
*******************************************************************************/
public DeleteResult execute(DeleteRequest deleteRequest) throws QException
{
try
{
DeleteResult rs = new DeleteResult();
QTableMetaData table = deleteRequest.getTable();
throw new NotImplementedException("Filesystem delete not implemented");
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing delete: " + e.getMessage(), e);
}
}
}

65
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java Normal file
View File

@ -0,0 +1,65 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class FilesystemInsertAction implements InsertInterface
{
/*******************************************************************************
**
*******************************************************************************/
public InsertResult execute(InsertRequest insertRequest) throws QException
{
try
{
InsertResult rs = new InsertResult();
QTableMetaData table = insertRequest.getTable();
List<QRecordWithStatus> recordsWithStatus = new ArrayList<>();
rs.setRecords(recordsWithStatus);
throw new NotImplementedException("Filesystem insert not implemented");
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
}
}

45
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryAction.java Normal file
View File

@ -0,0 +1,45 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
/*******************************************************************************
**
*******************************************************************************/
public class FilesystemQueryAction extends AbstractFilesystemAction implements QueryInterface
{
/*******************************************************************************
**
*******************************************************************************/
public QueryResult execute(QueryRequest queryRequest) throws QException
{
return executeQuery(queryRequest);
}
}

65
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemUpdateAction.java Normal file
View File

@ -0,0 +1,65 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class FilesystemUpdateAction implements UpdateInterface
{
/*******************************************************************************
**
*******************************************************************************/
public UpdateResult execute(UpdateRequest updateRequest) throws QException
{
try
{
UpdateResult rs = new UpdateResult();
QTableMetaData table = updateRequest.getTable();
List<QRecordWithStatus> recordsWithStatus = new ArrayList<>();
rs.setRecords(recordsWithStatus);
throw new NotImplementedException("Filesystem update not implemented");
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing update: " + e.getMessage(), e);
}
}
}

45
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java Normal file
View File

@ -0,0 +1,45 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule;
/*******************************************************************************
** (local) Filesystem backend meta data.
*******************************************************************************/
public class FilesystemBackendMetaData extends AbstractFilesystemBackendMetaData
{
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public FilesystemBackendMetaData()
{
super();
setBackendType(FilesystemBackendModule.class);
}
}

44
src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemTableBackendDetails.java Normal file
View File

@ -0,0 +1,44 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule;
/*******************************************************************************
** (local) Filesystem specific Extension of QTableBackendDetails
*******************************************************************************/
public class FilesystemTableBackendDetails extends AbstractFilesystemTableBackendDetails
{
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public FilesystemTableBackendDetails()
{
super();
setBackendType(FilesystemBackendModule.class);
}
}

121
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java Normal file
View File

@ -0,0 +1,121 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
/*******************************************************************************
** QQQ Backend module for working with AWS S3 filesystems
*******************************************************************************/
public class S3BackendModule implements QBackendModuleInterface
{
/*******************************************************************************
** Method where a backend module must be able to provide its type (name).
*******************************************************************************/
@Override
public String getBackendType()
{
return ("s3");
}
/*******************************************************************************
** Method to identify the class used for backend meta data for this module.
*******************************************************************************/
@Override
public Class<? extends QBackendMetaData> getBackendMetaDataClass()
{
return (S3BackendMetaData.class);
}
/*******************************************************************************
** Method to identify the class used for table-backend details for this module.
*******************************************************************************/
@Override
public Class<? extends QTableBackendDetails> getTableBackendDetailsClass()
{
return S3TableBackendDetails.class;
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QueryInterface getQueryInterface()
{
return new S3QueryAction();
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public InsertInterface getInsertInterface()
{
return (new S3InsertAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public UpdateInterface getUpdateInterface()
{
return (new S3UpdateAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public DeleteInterface getDeleteInterface()
{
return (new S3DeleteAction());
}
}

99
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java Normal file
View File

@ -0,0 +1,99 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
/*******************************************************************************
** Base class for all S3 filesystem actions
*******************************************************************************/
public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSummary>
{
private S3Utils s3Utils;
/*******************************************************************************
** Set the S3Utils object.
*******************************************************************************/
public void setS3Utils(S3Utils s3Utils)
{
this.s3Utils = s3Utils;
}
/*******************************************************************************
** Internal accessor for the s3Utils object - should always use this, not the field.
*******************************************************************************/
private S3Utils getS3Utils()
{
if(s3Utils == null)
{
s3Utils = new S3Utils();
}
return s3Utils;
}
/*******************************************************************************
** List the files for a table.
*******************************************************************************/
@Override
public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase)
{
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase);
String fullPath = getFullPath(table, backendBase);
String bucketName = s3BackendMetaData.getBucketName();
////////////////////////////////////////////////////////////////////
// todo - read metadata to decide if we should include subfolders //
// todo - look at metadata to configure the s3 client here? //
////////////////////////////////////////////////////////////////////
boolean includeSubfolders = false;
return getS3Utils().listObjectsInBucketAtPath(bucketName, fullPath, includeSubfolders);
}
/*******************************************************************************
** Read the contents of a file.
*******************************************************************************/
@Override
public InputStream readFile(S3ObjectSummary s3ObjectSummary) throws IOException
{
return (getS3Utils().getObjectAsInputStream(s3ObjectSummary));
}
}

59
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteAction.java Normal file
View File

@ -0,0 +1,59 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class S3DeleteAction implements DeleteInterface
{
/*******************************************************************************
**
*******************************************************************************/
public DeleteResult execute(DeleteRequest deleteRequest) throws QException
{
try
{
DeleteResult rs = new DeleteResult();
QTableMetaData table = deleteRequest.getTable();
throw new NotImplementedException("S3 delete not implemented");
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing delete: " + e.getMessage(), e);
}
}
}

65
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java Normal file
View File

@ -0,0 +1,65 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class S3InsertAction implements InsertInterface
{
/*******************************************************************************
**
*******************************************************************************/
public InsertResult execute(InsertRequest insertRequest) throws QException
{
try
{
InsertResult rs = new InsertResult();
QTableMetaData table = insertRequest.getTable();
List<QRecordWithStatus> recordsWithStatus = new ArrayList<>();
rs.setRecords(recordsWithStatus);
throw new NotImplementedException("S3 insert not implemented");
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
}
}

45
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryAction.java Normal file
View File

@ -0,0 +1,45 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
/*******************************************************************************
**
*******************************************************************************/
public class S3QueryAction extends AbstractS3Action implements QueryInterface
{
/*******************************************************************************
**
*******************************************************************************/
public QueryResult execute(QueryRequest queryRequest) throws QException
{
return (super.executeQuery(queryRequest));
}
}

65
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3UpdateAction.java Normal file
View File

@ -0,0 +1,65 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class S3UpdateAction implements UpdateInterface
{
/*******************************************************************************
**
*******************************************************************************/
public UpdateResult execute(UpdateRequest updateRequest) throws QException
{
try
{
UpdateResult rs = new UpdateResult();
QTableMetaData table = updateRequest.getTable();
List<QRecordWithStatus> recordsWithStatus = new ArrayList<>();
rs.setRecords(recordsWithStatus);
throw new NotImplementedException("S3 update not implemented");
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing update: " + e.getMessage(), e);
}
}
}

82
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java Normal file
View File

@ -0,0 +1,82 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
/*******************************************************************************
** (local) Filesystem backend meta data.
*******************************************************************************/
public class S3BackendMetaData extends AbstractFilesystemBackendMetaData
{
private String bucketName;
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public S3BackendMetaData()
{
super();
setBackendType(S3BackendModule.class);
}
/*******************************************************************************
** Getter for bucketName
**
*******************************************************************************/
public String getBucketName()
{
return bucketName;
}
/*******************************************************************************
** Setter for bucketName
**
*******************************************************************************/
public void setBucketName(String bucketName)
{
this.bucketName = bucketName;
}
/*******************************************************************************
** Fluent setter for bucketName
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public <T extends S3BackendMetaData> T withBucketName(String bucketName)
{
this.bucketName = bucketName;
return (T) this;
}
}

46
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3TableBackendDetails.java Normal file
View File

@ -0,0 +1,46 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
/*******************************************************************************
** S3 specific Extension of QTableBackendDetails
*******************************************************************************/
public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails
{
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public S3TableBackendDetails()
{
super();
setBackendType(S3BackendModule.class);
}
}

147
src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java Normal file
View File

@ -0,0 +1,147 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Utility methods for working with AWS S3.
*******************************************************************************/
public class S3Utils
{
private static final Logger LOG = LogManager.getLogger(S3Utils.class);
private AmazonS3 s3;
/*******************************************************************************
** List the objects in an S3 bucket at a given path
*******************************************************************************/
public List<S3ObjectSummary> listObjectsInBucketAtPath(String bucketName, String fullPath, boolean includeSubfolders)
{
//////////////////////////////////////////////////////////////////////////////////////////////////
// s3 list requests find nothing if the path starts with a /, so strip away any leading slashes //
// also strip away trailing /'s, for consistent known paths. //
// also normalize any duplicated /'s to a single /. //
//////////////////////////////////////////////////////////////////////////////////////////////////
fullPath = fullPath.replaceFirst("^/+", "").replaceFirst("/+$", "").replaceAll("//+", "/");
ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request()
.withBucketName(bucketName)
.withPrefix(fullPath);
ListObjectsV2Result listObjectsV2Result = null;
List<S3ObjectSummary> rs = new ArrayList<>();
do
{
if(listObjectsV2Result != null)
{
listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken());
}
listObjectsV2Result = getS3().listObjectsV2(listObjectsV2Request);
//////////////////////////////////
// put files in the result list //
//////////////////////////////////
for(S3ObjectSummary objectSummary : listObjectsV2Result.getObjectSummaries())
{
String key = objectSummary.getKey();
//////////////////
// skip folders //
//////////////////
if(key.endsWith("/"))
{
LOG.debug("Skipping file [{}] because it is a folder", key);
continue;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if we're not supposed to include subfolders, check the path on this file, and only include it if it matches the request //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(!includeSubfolders)
{
String prefix = key.substring(0, key.lastIndexOf("/"));
if(!prefix.equals(fullPath))
{
LOG.debug("Skipping file [{}] in a sub-folder [{}] != [{}]", key, prefix, fullPath);
continue;
}
}
rs.add(objectSummary);
}
}
while(listObjectsV2Result.isTruncated());
return rs;
}
/*******************************************************************************
**
*******************************************************************************/
public InputStream getObjectAsInputStream(S3ObjectSummary s3ObjectSummary)
{
return getS3().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()).getObjectContent();
}
/*******************************************************************************
** Setter for AmazonS3 client object.
*******************************************************************************/
public void setAmazonS3(AmazonS3 s3)
{
this.s3 = s3;
}
/*******************************************************************************
** Getter for AmazonS3 client object.
*******************************************************************************/
public AmazonS3 getS3()
{
if(s3 == null)
{
s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build();
}
return s3;
}
}

185
src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java Normal file
View File

@ -0,0 +1,185 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem;
import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import org.apache.commons.io.FileUtils;
/*******************************************************************************
** Utility methods for Filesystem module tests
*******************************************************************************/
public class TestUtils
{
private static int testInstanceCounter = 0;
private final static String BASE_PATH = "/tmp/filesystem-tests";
/*******************************************************************************
** Meant to be called in a @BeforeEach - increment an internal counter that will
** give us a unique directory name for each test method.
*******************************************************************************/
public static void increaseTestInstanceCounter()
{
testInstanceCounter++;
}
/*******************************************************************************
** Reset the counter to 0 (e.g., to let some tests have a known value).
*******************************************************************************/
public static void resetTestInstanceCounter()
{
testInstanceCounter = 0;
}
/*******************************************************************************
** Get the current value of the testInstanceCounter. See {@link #increaseTestInstanceCounter()}
*******************************************************************************/
public static int getTestInstanceCounter()
{
return (testInstanceCounter);
}
/*******************************************************************************
** Meant to run both after and before test methods - makes sure the file system
** is empty for the path under the instance.
*******************************************************************************/
public static void cleanInstanceFiles() throws IOException
{
FileUtils.deleteDirectory(new File(BASE_PATH + File.separator + testInstanceCounter));
}
/*******************************************************************************
**
*******************************************************************************/
public static QInstance defineInstance() throws QInstanceValidationException
{
QInstance qInstance = new QInstance();
qInstance.addBackend(defineLocalFilesystemBackend());
qInstance.addTable(defineLocalFilesystemCSVPersonTable());
qInstance.addBackend(defineS3Backend());
qInstance.addTable(defineS3CSVPersonTable());
new QInstanceValidator().validate(qInstance);
return (qInstance);
}
/*******************************************************************************
**
*******************************************************************************/
public static FilesystemBackendMetaData defineLocalFilesystemBackend()
{
return (new FilesystemBackendMetaData()
.withBasePath(BASE_PATH + File.separator + testInstanceCounter)
.withName("local-filesystem"));
}
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineLocalFilesystemCSVPersonTable()
{
return new QTableMetaData()
.withName("person")
.withLabel("Person")
.withBackendName(defineLocalFilesystemBackend().getName())
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"))
.withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"))
.withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"))
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
.withField(new QFieldMetaData("email", QFieldType.STRING))
.withBackendDetails(new FilesystemTableBackendDetails()
.withPath("persons")
.withRecordFormat("csv")
.withCardinality("many")
);
}
/*******************************************************************************
**
*******************************************************************************/
public static S3BackendMetaData defineS3Backend()
{
return (new S3BackendMetaData()
.withBucketName(BaseS3Test.BUCKET_NAME)
.withBasePath(BaseS3Test.TEST_FOLDER)
.withName("s3"));
}
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineS3CSVPersonTable()
{
return new QTableMetaData()
.withName("person-s3")
.withLabel("Person S3 Table")
.withBackendName(defineS3Backend().getName())
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"))
.withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"))
.withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"))
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
.withField(new QFieldMetaData("email", QFieldType.STRING))
.withBackendDetails(new S3TableBackendDetails()
.withRecordFormat("csv")
.withCardinality("many")
);
}
}

103
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java Normal file
View File

@ -0,0 +1,103 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.apache.commons.io.FileUtils;
import static org.junit.jupiter.api.Assertions.fail;
/*******************************************************************************
** Base class for Filesystem action tests.
**
** Knows how to set up the filesystem for the tests.
*******************************************************************************/
public class FilesystemActionTest
{
/*******************************************************************************
** Set up the file system
*******************************************************************************/
protected void primeFilesystem() throws IOException
{
TestUtils.cleanInstanceFiles();
TestUtils.increaseTestInstanceCounter();
FilesystemBackendMetaData filesystemBackendMetaData = TestUtils.defineLocalFilesystemBackend();
File baseDirectory = new File(filesystemBackendMetaData.getBasePath());
boolean mkdirsResult = baseDirectory.mkdirs();
if(!mkdirsResult)
{
fail("Failed to make directories at [" + baseDirectory + "] for filesystem backend module");
}
writePersonFiles(baseDirectory);
}
private void writePersonFiles(File baseDirectory) throws IOException
{
String fullPath = baseDirectory.getAbsolutePath();
if (TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
{
if (StringUtils.hasContent(details.getPath()))
{
fullPath += File.separatorChar + details.getPath();
}
}
fullPath += File.separatorChar;
String csvHeader = """
"id","createDate","modifyDate","firstName","lastName","birthDate","email"
""";
String csvData1 = csvHeader + """
"1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com"
"2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com"
""";
FileUtils.writeStringToFile(new File(fullPath + "DATA-1.csv"), csvData1);
String csvData2 = csvHeader + """
"3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com"
"4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com"
"5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com"
""";
FileUtils.writeStringToFile(new File(fullPath + "DATA-2.csv"), csvData2);
}
/*******************************************************************************
**
*******************************************************************************/
protected void cleanFilesystem() throws IOException
{
TestUtils.cleanInstanceFiles();
}
}

88
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java Normal file
View File

@ -0,0 +1,88 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/*******************************************************************************
**
*******************************************************************************/
public class FilesystemQueryActionTest extends FilesystemActionTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
public void beforeEach() throws Exception
{
super.primeFilesystem();
}
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
public void afterEach() throws Exception
{
super.cleanFilesystem();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQuery1() throws QException
{
QueryRequest queryRequest = initQueryRequest();
QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
Assertions.assertEquals(5, queryResult.getRecords().size(), "Unfiltered query should find all rows");
}
/*******************************************************************************
**
*******************************************************************************/
private QueryRequest initQueryRequest() throws QInstanceValidationException
{
QueryRequest queryRequest = new QueryRequest();
queryRequest.setInstance(TestUtils.defineInstance());
queryRequest.setTableName(TestUtils.defineLocalFilesystemCSVPersonTable().getName());
return queryRequest;
}
}

77
src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java Normal file
View File

@ -0,0 +1,77 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for FilesystemBackendMetaData
*******************************************************************************/
class FilesystemBackendMetaDataTest
{
/*******************************************************************************
** Test that an instance can be serialized as expected
*******************************************************************************/
@Test
public void testSerializingToJson() throws QInstanceValidationException
{
TestUtils.resetTestInstanceCounter();
QInstance qInstance = TestUtils.defineInstance();
String json = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance);
System.out.println(JsonUtils.prettyPrint(json));
System.out.println(json);
String expectToContain = """
"local-filesystem":{"basePath":"/tmp/filesystem-tests/0","backendType":"filesystem","name":"local-filesystem"}""";
assertTrue(json.contains(expectToContain));
}
/*******************************************************************************
** Test that an instance can be deserialized as expected
*******************************************************************************/
@Test
public void testDeserializingFromJson() throws IOException, QInstanceValidationException
{
QInstanceAdapter qInstanceAdapter = new QInstanceAdapter();
QInstance qInstance = TestUtils.defineInstance();
String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance);
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
assertThat(deserialized).usingRecursiveComparison()
.ignoringFields("hasBeenValidated")
.isEqualTo(qInstance);
}
}

173
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java Normal file
View File

@ -0,0 +1,173 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.testcontainers.containers.localstack.LocalStackContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;
/*******************************************************************************
**
*******************************************************************************/
@Testcontainers
public class BaseS3Test
{
private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("localstack/localstack");
public static final String BUCKET_NAME = "localstack-test-bucket";
public static final String TEST_FOLDER = "test-files";
public static final String SUB_FOLDER = "sub-folder";
@Container
private static LocalStackContainer localStack = new LocalStackContainer(DEFAULT_IMAGE_NAME)
.withServices(LocalStackContainer.Service.S3);
/*******************************************************************************
** Before each unit test, get the test bucket into a known state
*******************************************************************************/
@BeforeEach
public void beforeEach()
{
AmazonS3 amazonS3 = getAmazonS3();
amazonS3.createBucket(BUCKET_NAME);
amazonS3.putObject(BUCKET_NAME, "0.csv", getCSVHeader());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/1.csv", getCSVData1());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
}
/*******************************************************************************
** After each unit test, clean up the bucket
*******************************************************************************/
@AfterEach
public void afterEach()
{
AmazonS3 amazonS3 = getAmazonS3();
if(amazonS3.doesBucketExistV2(BUCKET_NAME))
{
////////////////////////
// todo - paginate... //
////////////////////////
for(S3ObjectSummary objectSummary : amazonS3.listObjectsV2(BUCKET_NAME).getObjectSummaries())
{
amazonS3.deleteObject(BUCKET_NAME, objectSummary.getKey());
}
amazonS3.deleteBucket(BUCKET_NAME);
}
}
/*******************************************************************************
** Access a localstack-configured s3 client.
*******************************************************************************/
protected AmazonS3 getAmazonS3()
{
BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(localStack.getAccessKey(), localStack.getSecretKey());
AmazonS3 amazonS3 = AmazonS3ClientBuilder.standard()
.withEndpointConfiguration(localStack.getEndpointConfiguration(LocalStackContainer.Service.S3))
.withCredentials(new AWSStaticCredentialsProvider(basicAWSCredentials))
.build();
return (amazonS3);
}
/*******************************************************************************
** Access the S3Utils object, with localstack-configured s3 client.
*******************************************************************************/
protected S3Utils getS3Utils()
{
S3Utils s3Utils = new S3Utils();
s3Utils.setAmazonS3(getAmazonS3());
return (s3Utils);
}
/*******************************************************************************
** Access a string of CSV test data.
*******************************************************************************/
protected String getCSVHeader()
{
return ("""
"id","createDate","modifyDate","firstName","lastName","birthDate","email"
""");
}
/*******************************************************************************
** Access a string of CSV test data.
*******************************************************************************/
protected String getCSVData1()
{
return (getCSVHeader() + """
"1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com"
"2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com"
""");
}
/*******************************************************************************
** Access a string of CSV test data.
*******************************************************************************/
protected String getCSVData2()
{
return (getCSVHeader() + """
"3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com"
"4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com"
"5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com"
""");
}
/*******************************************************************************
** Access a string of CSV test data.
*******************************************************************************/
protected String getCSVData3()
{
return (getCSVHeader() + """
"6","2022-06-20 15:31:02","2022-06-20 15:31:02","Lisa","S","1986-06-06","lisa.s@kingsrook.com"
""");
}
}

67
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java Normal file
View File

@ -0,0 +1,67 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/*******************************************************************************
**
*******************************************************************************/
public class S3QueryActionTest extends BaseS3Test
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQuery1() throws QException
{
QueryRequest queryRequest = initQueryRequest();
S3QueryAction s3QueryAction = new S3QueryAction();
s3QueryAction.setS3Utils(getS3Utils());
QueryResult queryResult = s3QueryAction.execute(queryRequest);
Assertions.assertEquals(5, queryResult.getRecords().size(), "Expected # of rows from unfiltered query");
}
/*******************************************************************************
**
*******************************************************************************/
private QueryRequest initQueryRequest() throws QInstanceValidationException
{
QueryRequest queryRequest = new QueryRequest();
queryRequest.setInstance(TestUtils.defineInstance());
queryRequest.setTableName(TestUtils.defineS3CSVPersonTable().getName());
return queryRequest;
}
}

77
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java Normal file
View File

@ -0,0 +1,77 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for S3BackendMetaData
*******************************************************************************/
class S3BackendMetaDataTest
{
/*******************************************************************************
** Test that an instance can be serialized as expected
*******************************************************************************/
@Test
public void testSerializingToJson() throws QInstanceValidationException
{
TestUtils.resetTestInstanceCounter();
QInstance qInstance = TestUtils.defineInstance();
String json = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance);
System.out.println(JsonUtils.prettyPrint(json));
System.out.println(json);
String expectToContain = """
{"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","backendType":"s3","name":"s3"}""";
assertTrue(json.contains(expectToContain));
}
/*******************************************************************************
** Test that an instance can be deserialized as expected
*******************************************************************************/
@Test
public void testDeserializingFromJson() throws IOException, QInstanceValidationException
{
QInstanceAdapter qInstanceAdapter = new QInstanceAdapter();
QInstance qInstance = TestUtils.defineInstance();
String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance);
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
assertThat(deserialized).usingRecursiveComparison()
.ignoringFields("hasBeenValidated")
.isEqualTo(qInstance);
}
}

75
src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java Normal file
View File

@ -0,0 +1,75 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
import java.io.IOException;
import java.io.InputStream;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
**
*******************************************************************************/
public class S3UtilsTest extends BaseS3Test
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testListObjectsInBucketAtPath()
{
S3Utils s3Utils = getS3Utils();
assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/", false).size(), "Expected # of s3 objects without subfolders");
assertEquals(3, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/", true).size(), "Expected # of s3 objects with subfolders");
assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "/" + TEST_FOLDER + "/", false).size(), "With leading slash");
assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "/" + TEST_FOLDER, false).size(), "Without trailing slash");
assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "//" + TEST_FOLDER + "//", false).size(), "With multiple leading and trailing slashes");
assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, false).size(), "Just in the subfolder non-recursive");
assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, true).size(), "Just in the subfolder recursive");
assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "//" + SUB_FOLDER, true).size(), "Just in the subfolder recursive");
assertEquals(0, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "not-a-real-path/", true).size(), "In a non-existing folder");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testGetObjectAsInputStream() throws IOException
{
S3Utils s3Utils = getS3Utils();
S3ObjectSummary s3ObjectSummary = s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "test-files", true).get(0);
InputStream inputStream = s3Utils.getObjectAsInputStream(s3ObjectSummary);
String csvFromS3 = IOUtils.toString(inputStream);
// todo - should check the filename somewhere, right?
assertEquals(getCSVData1(), csvFromS3, "File from S3 should match expected content");
}
}