mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 05:01:07 +00:00
QQQ-14 initial checkin
This commit is contained in:
@ -0,0 +1,163 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.base.actions;
|
||||
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
|
||||
import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang.NotImplementedException;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Base class for all Filesystem actions across all modules.
|
||||
*******************************************************************************/
|
||||
public abstract class AbstractBaseFilesystemAction<FILE>
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** List the files for a table - to be implemented in module-specific subclasses.
|
||||
*******************************************************************************/
|
||||
public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase);
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the contents of a file - to be implemented in module-specific subclasses.
|
||||
*******************************************************************************/
|
||||
public abstract InputStream readFile(FILE file) throws IOException;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Append together the backend's base path (if present), with a table's path (again, if present).
|
||||
*******************************************************************************/
|
||||
protected String getFullPath(QTableMetaData table, QBackendMetaData backendBase)
|
||||
{
|
||||
AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase);
|
||||
String fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : "";
|
||||
|
||||
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
|
||||
if(StringUtils.hasContent(tableDetails.getPath()))
|
||||
{
|
||||
fullPath += File.separatorChar + tableDetails.getPath();
|
||||
}
|
||||
|
||||
fullPath += File.separatorChar;
|
||||
return fullPath;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Get the backend metaData, type-checked as the requested type.
|
||||
*******************************************************************************/
|
||||
protected <T extends AbstractFilesystemBackendMetaData> T getBackendMetaData(Class<T> outputClass, QBackendMetaData metaData)
|
||||
{
|
||||
if(!(outputClass.isInstance(metaData)))
|
||||
{
|
||||
throw new IllegalArgumentException("MetaData was not of expected type (was " + metaData.getClass().getSimpleName() + ")");
|
||||
}
|
||||
return outputClass.cast(metaData);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Get the backendDetails out of a table, type-checked as the requested type
|
||||
*******************************************************************************/
|
||||
protected <T extends AbstractFilesystemTableBackendDetails> T getTableBackendDetails(Class<T> outputClass, QTableMetaData tableMetaData)
|
||||
{
|
||||
QTableBackendDetails tableBackendDetails = tableMetaData.getBackendDetails();
|
||||
if(!(outputClass.isInstance(tableBackendDetails)))
|
||||
{
|
||||
throw new IllegalArgumentException("Table backend details was not of expected type (was " + tableBackendDetails.getClass().getSimpleName() + ")");
|
||||
}
|
||||
return outputClass.cast(tableBackendDetails);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Generic implementation of the execute method from the QueryInterface
|
||||
*******************************************************************************/
|
||||
public QueryResult executeQuery(QueryRequest queryRequest) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
QueryResult rs = new QueryResult();
|
||||
List<QRecord> records = new ArrayList<>();
|
||||
rs.setRecords(records);
|
||||
|
||||
QTableMetaData table = queryRequest.getTable();
|
||||
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
|
||||
List<FILE> files = listFiles(table, queryRequest.getBackend());
|
||||
|
||||
for(FILE file : files)
|
||||
{
|
||||
switch(tableDetails.getRecordFormat())
|
||||
{
|
||||
case "csv":
|
||||
{
|
||||
String fileContents = IOUtils.toString(readFile(file));
|
||||
List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
|
||||
|
||||
records.addAll(recordsInFile);
|
||||
break;
|
||||
}
|
||||
case "json":
|
||||
{
|
||||
String fileContents = IOUtils.toString(readFile(file));
|
||||
List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
|
||||
|
||||
records.addAll(recordsInFile);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
{
|
||||
throw new NotImplementedException("Filesystem record format " + tableDetails.getRecordFormat() + " is not yet implemented");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return rs;
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
e.printStackTrace();
|
||||
throw new QException("Error executing query", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,80 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Base class for all BackendMetaData for all filesystem-style backend modules.
|
||||
*******************************************************************************/
|
||||
public class AbstractFilesystemBackendMetaData extends QBackendMetaData
|
||||
{
|
||||
private String basePath;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Default Constructor.
|
||||
*******************************************************************************/
|
||||
public AbstractFilesystemBackendMetaData()
|
||||
{
|
||||
super();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for basePath
|
||||
**
|
||||
*******************************************************************************/
|
||||
public String getBasePath()
|
||||
{
|
||||
return (basePath);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for basePath
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void setBasePath(String basePath)
|
||||
{
|
||||
this.basePath = basePath;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for basePath
|
||||
**
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
public <T extends AbstractFilesystemBackendMetaData> T withBasePath(String basePath)
|
||||
{
|
||||
this.basePath = basePath;
|
||||
return (T) this;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,142 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Extension of QTableBackendDetails, with details specific to a Filesystem table.
|
||||
*******************************************************************************/
|
||||
public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
|
||||
{
|
||||
private String path;
|
||||
private String recordFormat; // todo - enum? but hard w/ serialization?
|
||||
private String cardinality; // todo - enum?
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for path
|
||||
**
|
||||
*******************************************************************************/
|
||||
public String getPath()
|
||||
{
|
||||
return path;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for path
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void setPath(String path)
|
||||
{
|
||||
this.path = path;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent Setter for path
|
||||
**
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
public <T extends AbstractFilesystemTableBackendDetails> T withPath(String path)
|
||||
{
|
||||
this.path = path;
|
||||
return (T) this;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for recordFormat
|
||||
**
|
||||
*******************************************************************************/
|
||||
public String getRecordFormat()
|
||||
{
|
||||
return recordFormat;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for recordFormat
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void setRecordFormat(String recordFormat)
|
||||
{
|
||||
this.recordFormat = recordFormat;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent Setter for recordFormat
|
||||
**
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
public <T extends AbstractFilesystemTableBackendDetails> T withRecordFormat(String recordFormat)
|
||||
{
|
||||
this.recordFormat = recordFormat;
|
||||
return ((T) this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for cardinality
|
||||
**
|
||||
*******************************************************************************/
|
||||
public String getCardinality()
|
||||
{
|
||||
return cardinality;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for cardinality
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void setCardinality(String cardinality)
|
||||
{
|
||||
this.cardinality = cardinality;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent Setter for cardinality
|
||||
**
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
public <T extends AbstractFilesystemTableBackendDetails> T withCardinality(String cardinality)
|
||||
{
|
||||
this.cardinality = cardinality;
|
||||
return ((T) this);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,119 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package com.kingsrook.qqq.backend.module.filesystem.local;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemUpdateAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** QQQ Backend module for working with (local) Filesystems.
|
||||
*******************************************************************************/
|
||||
public class FilesystemBackendModule implements QBackendModuleInterface
|
||||
{
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Method where a backend module must be able to provide its type (name).
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getBackendType()
|
||||
{
|
||||
return ("filesystem");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Method to identify the class used for backend meta data for this module.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Class<? extends QBackendMetaData> getBackendMetaDataClass()
|
||||
{
|
||||
return (FilesystemBackendMetaData.class);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Method to identify the class used for table-backend details for this module.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Class<? extends QTableBackendDetails> getTableBackendDetailsClass()
|
||||
{
|
||||
return FilesystemTableBackendDetails.class;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QueryInterface getQueryInterface()
|
||||
{
|
||||
return new FilesystemQueryAction();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public InsertInterface getInsertInterface()
|
||||
{
|
||||
return (new FilesystemInsertAction());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public UpdateInterface getUpdateInterface()
|
||||
{
|
||||
return (new FilesystemUpdateAction());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public DeleteInterface getDeleteInterface()
|
||||
{
|
||||
return (new FilesystemDeleteAction());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,64 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
||||
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Base class for all (local) Filesystem actions
|
||||
*******************************************************************************/
|
||||
public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** List the files for this table.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase)
|
||||
{
|
||||
String fullPath = getFullPath(table, backendBase);
|
||||
File directory = new File(fullPath);
|
||||
return Arrays.asList(directory.listFiles());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the contents of a file.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public InputStream readFile(File file) throws IOException
|
||||
{
|
||||
return (new FileInputStream(file));
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,59 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
|
||||
import org.apache.commons.lang.NotImplementedException;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public class FilesystemDeleteAction implements DeleteInterface
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public DeleteResult execute(DeleteRequest deleteRequest) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
DeleteResult rs = new DeleteResult();
|
||||
QTableMetaData table = deleteRequest.getTable();
|
||||
|
||||
throw new NotImplementedException("Filesystem delete not implemented");
|
||||
|
||||
// return rs;
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw new QException("Error executing delete: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,65 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
|
||||
import org.apache.commons.lang.NotImplementedException;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public class FilesystemInsertAction implements InsertInterface
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public InsertResult execute(InsertRequest insertRequest) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
InsertResult rs = new InsertResult();
|
||||
QTableMetaData table = insertRequest.getTable();
|
||||
|
||||
List<QRecordWithStatus> recordsWithStatus = new ArrayList<>();
|
||||
rs.setRecords(recordsWithStatus);
|
||||
|
||||
throw new NotImplementedException("Filesystem insert not implemented");
|
||||
|
||||
// return rs;
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw new QException("Error executing insert: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,45 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public class FilesystemQueryAction extends AbstractFilesystemAction implements QueryInterface
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public QueryResult execute(QueryRequest queryRequest) throws QException
|
||||
{
|
||||
return executeQuery(queryRequest);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,65 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
|
||||
import org.apache.commons.lang.NotImplementedException;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public class FilesystemUpdateAction implements UpdateInterface
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public UpdateResult execute(UpdateRequest updateRequest) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
UpdateResult rs = new UpdateResult();
|
||||
QTableMetaData table = updateRequest.getTable();
|
||||
|
||||
List<QRecordWithStatus> recordsWithStatus = new ArrayList<>();
|
||||
rs.setRecords(recordsWithStatus);
|
||||
|
||||
throw new NotImplementedException("Filesystem update not implemented");
|
||||
|
||||
// return rs;
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw new QException("Error executing update: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,45 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** (local) Filesystem backend meta data.
|
||||
*******************************************************************************/
|
||||
public class FilesystemBackendMetaData extends AbstractFilesystemBackendMetaData
|
||||
{
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Default Constructor.
|
||||
*******************************************************************************/
|
||||
public FilesystemBackendMetaData()
|
||||
{
|
||||
super();
|
||||
setBackendType(FilesystemBackendModule.class);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,44 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** (local) Filesystem specific Extension of QTableBackendDetails
|
||||
*******************************************************************************/
|
||||
public class FilesystemTableBackendDetails extends AbstractFilesystemTableBackendDetails
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** Default Constructor.
|
||||
*******************************************************************************/
|
||||
public FilesystemTableBackendDetails()
|
||||
{
|
||||
super();
|
||||
setBackendType(FilesystemBackendModule.class);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,121 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** QQQ Backend module for working with AWS S3 filesystems
|
||||
*******************************************************************************/
|
||||
public class S3BackendModule implements QBackendModuleInterface
|
||||
{
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Method where a backend module must be able to provide its type (name).
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getBackendType()
|
||||
{
|
||||
return ("s3");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Method to identify the class used for backend meta data for this module.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Class<? extends QBackendMetaData> getBackendMetaDataClass()
|
||||
{
|
||||
return (S3BackendMetaData.class);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Method to identify the class used for table-backend details for this module.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Class<? extends QTableBackendDetails> getTableBackendDetailsClass()
|
||||
{
|
||||
return S3TableBackendDetails.class;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QueryInterface getQueryInterface()
|
||||
{
|
||||
return new S3QueryAction();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public InsertInterface getInsertInterface()
|
||||
{
|
||||
return (new S3InsertAction());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public UpdateInterface getUpdateInterface()
|
||||
{
|
||||
return (new S3UpdateAction());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public DeleteInterface getDeleteInterface()
|
||||
{
|
||||
return (new S3DeleteAction());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,99 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.List;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Base class for all S3 filesystem actions
|
||||
*******************************************************************************/
|
||||
public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSummary>
|
||||
{
|
||||
private S3Utils s3Utils;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Set the S3Utils object.
|
||||
*******************************************************************************/
|
||||
public void setS3Utils(S3Utils s3Utils)
|
||||
{
|
||||
this.s3Utils = s3Utils;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Internal accessor for the s3Utils object - should always use this, not the field.
|
||||
*******************************************************************************/
|
||||
private S3Utils getS3Utils()
|
||||
{
|
||||
if(s3Utils == null)
|
||||
{
|
||||
s3Utils = new S3Utils();
|
||||
}
|
||||
|
||||
return s3Utils;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** List the files for a table.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase)
|
||||
{
|
||||
S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase);
|
||||
|
||||
String fullPath = getFullPath(table, backendBase);
|
||||
String bucketName = s3BackendMetaData.getBucketName();
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// todo - read metadata to decide if we should include subfolders //
|
||||
// todo - look at metadata to configure the s3 client here? //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
boolean includeSubfolders = false;
|
||||
return getS3Utils().listObjectsInBucketAtPath(bucketName, fullPath, includeSubfolders);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the contents of a file.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public InputStream readFile(S3ObjectSummary s3ObjectSummary) throws IOException
|
||||
{
|
||||
return (getS3Utils().getObjectAsInputStream(s3ObjectSummary));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,59 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
|
||||
import org.apache.commons.lang.NotImplementedException;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public class S3DeleteAction implements DeleteInterface
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public DeleteResult execute(DeleteRequest deleteRequest) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
DeleteResult rs = new DeleteResult();
|
||||
QTableMetaData table = deleteRequest.getTable();
|
||||
|
||||
throw new NotImplementedException("S3 delete not implemented");
|
||||
|
||||
// return rs;
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw new QException("Error executing delete: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,65 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
|
||||
import org.apache.commons.lang.NotImplementedException;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public class S3InsertAction implements InsertInterface
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public InsertResult execute(InsertRequest insertRequest) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
InsertResult rs = new InsertResult();
|
||||
QTableMetaData table = insertRequest.getTable();
|
||||
|
||||
List<QRecordWithStatus> recordsWithStatus = new ArrayList<>();
|
||||
rs.setRecords(recordsWithStatus);
|
||||
|
||||
throw new NotImplementedException("S3 insert not implemented");
|
||||
|
||||
// return rs;
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw new QException("Error executing insert: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,45 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public class S3QueryAction extends AbstractS3Action implements QueryInterface
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public QueryResult execute(QueryRequest queryRequest) throws QException
|
||||
{
|
||||
return (super.executeQuery(queryRequest));
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,65 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecordWithStatus;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
|
||||
import org.apache.commons.lang.NotImplementedException;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public class S3UpdateAction implements UpdateInterface
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public UpdateResult execute(UpdateRequest updateRequest) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
UpdateResult rs = new UpdateResult();
|
||||
QTableMetaData table = updateRequest.getTable();
|
||||
|
||||
List<QRecordWithStatus> recordsWithStatus = new ArrayList<>();
|
||||
rs.setRecords(recordsWithStatus);
|
||||
|
||||
throw new NotImplementedException("S3 update not implemented");
|
||||
|
||||
// return rs;
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw new QException("Error executing update: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,82 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** (local) Filesystem backend meta data.
|
||||
*******************************************************************************/
|
||||
public class S3BackendMetaData extends AbstractFilesystemBackendMetaData
|
||||
{
|
||||
private String bucketName;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Default Constructor.
|
||||
*******************************************************************************/
|
||||
public S3BackendMetaData()
|
||||
{
|
||||
super();
|
||||
setBackendType(S3BackendModule.class);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for bucketName
|
||||
**
|
||||
*******************************************************************************/
|
||||
public String getBucketName()
|
||||
{
|
||||
return bucketName;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for bucketName
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void setBucketName(String bucketName)
|
||||
{
|
||||
this.bucketName = bucketName;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for bucketName
|
||||
**
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
public <T extends S3BackendMetaData> T withBucketName(String bucketName)
|
||||
{
|
||||
this.bucketName = bucketName;
|
||||
return (T) this;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,46 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** S3 specific Extension of QTableBackendDetails
|
||||
*******************************************************************************/
|
||||
public class S3TableBackendDetails extends AbstractFilesystemTableBackendDetails
|
||||
{
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Default Constructor.
|
||||
*******************************************************************************/
|
||||
public S3TableBackendDetails()
|
||||
{
|
||||
super();
|
||||
setBackendType(S3BackendModule.class);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,147 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
|
||||
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.amazonaws.regions.Regions;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||
import com.amazonaws.services.s3.model.ListObjectsV2Request;
|
||||
import com.amazonaws.services.s3.model.ListObjectsV2Result;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility methods for working with AWS S3.
|
||||
*******************************************************************************/
|
||||
public class S3Utils
|
||||
{
|
||||
private static final Logger LOG = LogManager.getLogger(S3Utils.class);
|
||||
|
||||
private AmazonS3 s3;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** List the objects in an S3 bucket at a given path
|
||||
*******************************************************************************/
|
||||
public List<S3ObjectSummary> listObjectsInBucketAtPath(String bucketName, String fullPath, boolean includeSubfolders)
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// s3 list requests find nothing if the path starts with a /, so strip away any leading slashes //
|
||||
// also strip away trailing /'s, for consistent known paths. //
|
||||
// also normalize any duplicated /'s to a single /. //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
fullPath = fullPath.replaceFirst("^/+", "").replaceFirst("/+$", "").replaceAll("//+", "/");
|
||||
|
||||
ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request()
|
||||
.withBucketName(bucketName)
|
||||
.withPrefix(fullPath);
|
||||
|
||||
ListObjectsV2Result listObjectsV2Result = null;
|
||||
List<S3ObjectSummary> rs = new ArrayList<>();
|
||||
|
||||
do
|
||||
{
|
||||
if(listObjectsV2Result != null)
|
||||
{
|
||||
listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken());
|
||||
}
|
||||
listObjectsV2Result = getS3().listObjectsV2(listObjectsV2Request);
|
||||
|
||||
//////////////////////////////////
|
||||
// put files in the result list //
|
||||
//////////////////////////////////
|
||||
for(S3ObjectSummary objectSummary : listObjectsV2Result.getObjectSummaries())
|
||||
{
|
||||
String key = objectSummary.getKey();
|
||||
|
||||
//////////////////
|
||||
// skip folders //
|
||||
//////////////////
|
||||
if(key.endsWith("/"))
|
||||
{
|
||||
LOG.debug("Skipping file [{}] because it is a folder", key);
|
||||
continue;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// if we're not supposed to include subfolders, check the path on this file, and only include it if it matches the request //
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(!includeSubfolders)
|
||||
{
|
||||
String prefix = key.substring(0, key.lastIndexOf("/"));
|
||||
if(!prefix.equals(fullPath))
|
||||
{
|
||||
LOG.debug("Skipping file [{}] in a sub-folder [{}] != [{}]", key, prefix, fullPath);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
rs.add(objectSummary);
|
||||
}
|
||||
}
|
||||
while(listObjectsV2Result.isTruncated());
|
||||
|
||||
return rs;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public InputStream getObjectAsInputStream(S3ObjectSummary s3ObjectSummary)
|
||||
{
|
||||
return getS3().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()).getObjectContent();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for AmazonS3 client object.
|
||||
*******************************************************************************/
|
||||
public void setAmazonS3(AmazonS3 s3)
|
||||
{
|
||||
this.s3 = s3;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for AmazonS3 client object.
|
||||
*******************************************************************************/
|
||||
public AmazonS3 getS3()
|
||||
{
|
||||
if(s3 == null)
|
||||
{
|
||||
s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build();
|
||||
}
|
||||
|
||||
return s3;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -0,0 +1,185 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem;
|
||||
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
|
||||
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility methods for Filesystem module tests
|
||||
*******************************************************************************/
|
||||
public class TestUtils
|
||||
{
|
||||
private static int testInstanceCounter = 0;
|
||||
private final static String BASE_PATH = "/tmp/filesystem-tests";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Meant to be called in a @BeforeEach - increment an internal counter that will
|
||||
** give us a unique directory name for each test method.
|
||||
*******************************************************************************/
|
||||
public static void increaseTestInstanceCounter()
|
||||
{
|
||||
testInstanceCounter++;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reset the counter to 0 (e.g., to let some tests have a known value).
|
||||
*******************************************************************************/
|
||||
public static void resetTestInstanceCounter()
|
||||
{
|
||||
testInstanceCounter = 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Get the current value of the testInstanceCounter. See {@link #increaseTestInstanceCounter()}
|
||||
*******************************************************************************/
|
||||
public static int getTestInstanceCounter()
|
||||
{
|
||||
return (testInstanceCounter);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Meant to run both after and before test methods - makes sure the file system
|
||||
** is empty for the path under the instance.
|
||||
*******************************************************************************/
|
||||
public static void cleanInstanceFiles() throws IOException
|
||||
{
|
||||
FileUtils.deleteDirectory(new File(BASE_PATH + File.separator + testInstanceCounter));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static QInstance defineInstance() throws QInstanceValidationException
|
||||
{
|
||||
QInstance qInstance = new QInstance();
|
||||
qInstance.addBackend(defineLocalFilesystemBackend());
|
||||
qInstance.addTable(defineLocalFilesystemCSVPersonTable());
|
||||
qInstance.addBackend(defineS3Backend());
|
||||
qInstance.addTable(defineS3CSVPersonTable());
|
||||
|
||||
new QInstanceValidator().validate(qInstance);
|
||||
|
||||
return (qInstance);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static FilesystemBackendMetaData defineLocalFilesystemBackend()
|
||||
{
|
||||
return (new FilesystemBackendMetaData()
|
||||
.withBasePath(BASE_PATH + File.separator + testInstanceCounter)
|
||||
.withName("local-filesystem"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static QTableMetaData defineLocalFilesystemCSVPersonTable()
|
||||
{
|
||||
return new QTableMetaData()
|
||||
.withName("person")
|
||||
.withLabel("Person")
|
||||
.withBackendName(defineLocalFilesystemBackend().getName())
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"))
|
||||
.withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"))
|
||||
.withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"))
|
||||
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
|
||||
.withField(new QFieldMetaData("email", QFieldType.STRING))
|
||||
.withBackendDetails(new FilesystemTableBackendDetails()
|
||||
.withPath("persons")
|
||||
.withRecordFormat("csv")
|
||||
.withCardinality("many")
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static S3BackendMetaData defineS3Backend()
|
||||
{
|
||||
return (new S3BackendMetaData()
|
||||
.withBucketName(BaseS3Test.BUCKET_NAME)
|
||||
.withBasePath(BaseS3Test.TEST_FOLDER)
|
||||
.withName("s3"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static QTableMetaData defineS3CSVPersonTable()
|
||||
{
|
||||
return new QTableMetaData()
|
||||
.withName("person-s3")
|
||||
.withLabel("Person S3 Table")
|
||||
.withBackendName(defineS3Backend().getName())
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"))
|
||||
.withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"))
|
||||
.withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"))
|
||||
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
|
||||
.withField(new QFieldMetaData("email", QFieldType.STRING))
|
||||
.withBackendDetails(new S3TableBackendDetails()
|
||||
.withRecordFormat("csv")
|
||||
.withCardinality("many")
|
||||
);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,103 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
||||
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Base class for Filesystem action tests.
|
||||
**
|
||||
** Knows how to set up the filesystem for the tests.
|
||||
*******************************************************************************/
|
||||
public class FilesystemActionTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** Set up the file system
|
||||
*******************************************************************************/
|
||||
protected void primeFilesystem() throws IOException
|
||||
{
|
||||
TestUtils.cleanInstanceFiles();
|
||||
TestUtils.increaseTestInstanceCounter();
|
||||
FilesystemBackendMetaData filesystemBackendMetaData = TestUtils.defineLocalFilesystemBackend();
|
||||
|
||||
File baseDirectory = new File(filesystemBackendMetaData.getBasePath());
|
||||
boolean mkdirsResult = baseDirectory.mkdirs();
|
||||
if(!mkdirsResult)
|
||||
{
|
||||
fail("Failed to make directories at [" + baseDirectory + "] for filesystem backend module");
|
||||
}
|
||||
|
||||
writePersonFiles(baseDirectory);
|
||||
}
|
||||
|
||||
|
||||
|
||||
private void writePersonFiles(File baseDirectory) throws IOException
|
||||
{
|
||||
String fullPath = baseDirectory.getAbsolutePath();
|
||||
if (TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
|
||||
{
|
||||
if (StringUtils.hasContent(details.getPath()))
|
||||
{
|
||||
fullPath += File.separatorChar + details.getPath();
|
||||
}
|
||||
}
|
||||
fullPath += File.separatorChar;
|
||||
|
||||
String csvHeader = """
|
||||
"id","createDate","modifyDate","firstName","lastName","birthDate","email"
|
||||
""";
|
||||
|
||||
String csvData1 = csvHeader + """
|
||||
"1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com"
|
||||
"2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com"
|
||||
""";
|
||||
FileUtils.writeStringToFile(new File(fullPath + "DATA-1.csv"), csvData1);
|
||||
|
||||
String csvData2 = csvHeader + """
|
||||
"3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com"
|
||||
"4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com"
|
||||
"5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com"
|
||||
""";
|
||||
FileUtils.writeStringToFile(new File(fullPath + "DATA-2.csv"), csvData2);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
protected void cleanFilesystem() throws IOException
|
||||
{
|
||||
TestUtils.cleanInstanceFiles();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,88 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public class FilesystemQueryActionTest extends FilesystemActionTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
public void beforeEach() throws Exception
|
||||
{
|
||||
super.primeFilesystem();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
public void afterEach() throws Exception
|
||||
{
|
||||
super.cleanFilesystem();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testQuery1() throws QException
|
||||
{
|
||||
QueryRequest queryRequest = initQueryRequest();
|
||||
QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
|
||||
Assertions.assertEquals(5, queryResult.getRecords().size(), "Unfiltered query should find all rows");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private QueryRequest initQueryRequest() throws QInstanceValidationException
|
||||
{
|
||||
QueryRequest queryRequest = new QueryRequest();
|
||||
queryRequest.setInstance(TestUtils.defineInstance());
|
||||
queryRequest.setTableName(TestUtils.defineLocalFilesystemCSVPersonTable().getName());
|
||||
return queryRequest;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,77 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for FilesystemBackendMetaData
|
||||
*******************************************************************************/
|
||||
class FilesystemBackendMetaDataTest
|
||||
{
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that an instance can be serialized as expected
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testSerializingToJson() throws QInstanceValidationException
|
||||
{
|
||||
TestUtils.resetTestInstanceCounter();
|
||||
QInstance qInstance = TestUtils.defineInstance();
|
||||
String json = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance);
|
||||
System.out.println(JsonUtils.prettyPrint(json));
|
||||
System.out.println(json);
|
||||
String expectToContain = """
|
||||
"local-filesystem":{"basePath":"/tmp/filesystem-tests/0","backendType":"filesystem","name":"local-filesystem"}""";
|
||||
assertTrue(json.contains(expectToContain));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that an instance can be deserialized as expected
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testDeserializingFromJson() throws IOException, QInstanceValidationException
|
||||
{
|
||||
QInstanceAdapter qInstanceAdapter = new QInstanceAdapter();
|
||||
|
||||
QInstance qInstance = TestUtils.defineInstance();
|
||||
String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance);
|
||||
|
||||
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
|
||||
assertThat(deserialized).usingRecursiveComparison()
|
||||
.ignoringFields("hasBeenValidated")
|
||||
.isEqualTo(qInstance);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,173 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3;
|
||||
|
||||
|
||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.testcontainers.containers.localstack.LocalStackContainer;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Testcontainers
|
||||
public class BaseS3Test
|
||||
{
|
||||
private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("localstack/localstack");
|
||||
|
||||
public static final String BUCKET_NAME = "localstack-test-bucket";
|
||||
public static final String TEST_FOLDER = "test-files";
|
||||
public static final String SUB_FOLDER = "sub-folder";
|
||||
|
||||
@Container
|
||||
private static LocalStackContainer localStack = new LocalStackContainer(DEFAULT_IMAGE_NAME)
|
||||
.withServices(LocalStackContainer.Service.S3);
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Before each unit test, get the test bucket into a known state
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
public void beforeEach()
|
||||
{
|
||||
AmazonS3 amazonS3 = getAmazonS3();
|
||||
|
||||
amazonS3.createBucket(BUCKET_NAME);
|
||||
amazonS3.putObject(BUCKET_NAME, "0.csv", getCSVHeader());
|
||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/1.csv", getCSVData1());
|
||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2());
|
||||
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** After each unit test, clean up the bucket
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
public void afterEach()
|
||||
{
|
||||
AmazonS3 amazonS3 = getAmazonS3();
|
||||
|
||||
if(amazonS3.doesBucketExistV2(BUCKET_NAME))
|
||||
{
|
||||
////////////////////////
|
||||
// todo - paginate... //
|
||||
////////////////////////
|
||||
for(S3ObjectSummary objectSummary : amazonS3.listObjectsV2(BUCKET_NAME).getObjectSummaries())
|
||||
{
|
||||
amazonS3.deleteObject(BUCKET_NAME, objectSummary.getKey());
|
||||
}
|
||||
amazonS3.deleteBucket(BUCKET_NAME);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access a localstack-configured s3 client.
|
||||
*******************************************************************************/
|
||||
protected AmazonS3 getAmazonS3()
|
||||
{
|
||||
BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(localStack.getAccessKey(), localStack.getSecretKey());
|
||||
AmazonS3 amazonS3 = AmazonS3ClientBuilder.standard()
|
||||
.withEndpointConfiguration(localStack.getEndpointConfiguration(LocalStackContainer.Service.S3))
|
||||
.withCredentials(new AWSStaticCredentialsProvider(basicAWSCredentials))
|
||||
.build();
|
||||
return (amazonS3);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access the S3Utils object, with localstack-configured s3 client.
|
||||
*******************************************************************************/
|
||||
protected S3Utils getS3Utils()
|
||||
{
|
||||
S3Utils s3Utils = new S3Utils();
|
||||
s3Utils.setAmazonS3(getAmazonS3());
|
||||
return (s3Utils);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access a string of CSV test data.
|
||||
*******************************************************************************/
|
||||
protected String getCSVHeader()
|
||||
{
|
||||
return ("""
|
||||
"id","createDate","modifyDate","firstName","lastName","birthDate","email"
|
||||
""");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access a string of CSV test data.
|
||||
*******************************************************************************/
|
||||
protected String getCSVData1()
|
||||
{
|
||||
return (getCSVHeader() + """
|
||||
"1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com"
|
||||
"2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com"
|
||||
""");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access a string of CSV test data.
|
||||
*******************************************************************************/
|
||||
protected String getCSVData2()
|
||||
{
|
||||
return (getCSVHeader() + """
|
||||
"3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com"
|
||||
"4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com"
|
||||
"5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com"
|
||||
""");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access a string of CSV test data.
|
||||
*******************************************************************************/
|
||||
protected String getCSVData3()
|
||||
{
|
||||
return (getCSVHeader() + """
|
||||
"6","2022-06-20 15:31:02","2022-06-20 15:31:02","Lisa","S","1986-06-06","lisa.s@kingsrook.com"
|
||||
""");
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,67 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public class S3QueryActionTest extends BaseS3Test
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testQuery1() throws QException
|
||||
{
|
||||
QueryRequest queryRequest = initQueryRequest();
|
||||
S3QueryAction s3QueryAction = new S3QueryAction();
|
||||
s3QueryAction.setS3Utils(getS3Utils());
|
||||
QueryResult queryResult = s3QueryAction.execute(queryRequest);
|
||||
Assertions.assertEquals(5, queryResult.getRecords().size(), "Expected # of rows from unfiltered query");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private QueryRequest initQueryRequest() throws QInstanceValidationException
|
||||
{
|
||||
QueryRequest queryRequest = new QueryRequest();
|
||||
queryRequest.setInstance(TestUtils.defineInstance());
|
||||
queryRequest.setTableName(TestUtils.defineS3CSVPersonTable().getName());
|
||||
return queryRequest;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,77 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for S3BackendMetaData
|
||||
*******************************************************************************/
|
||||
class S3BackendMetaDataTest
|
||||
{
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that an instance can be serialized as expected
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testSerializingToJson() throws QInstanceValidationException
|
||||
{
|
||||
TestUtils.resetTestInstanceCounter();
|
||||
QInstance qInstance = TestUtils.defineInstance();
|
||||
String json = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance);
|
||||
System.out.println(JsonUtils.prettyPrint(json));
|
||||
System.out.println(json);
|
||||
String expectToContain = """
|
||||
{"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","backendType":"s3","name":"s3"}""";
|
||||
assertTrue(json.contains(expectToContain));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that an instance can be deserialized as expected
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testDeserializingFromJson() throws IOException, QInstanceValidationException
|
||||
{
|
||||
QInstanceAdapter qInstanceAdapter = new QInstanceAdapter();
|
||||
|
||||
QInstance qInstance = TestUtils.defineInstance();
|
||||
String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance);
|
||||
|
||||
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
|
||||
assertThat(deserialized).usingRecursiveComparison()
|
||||
.ignoringFields("hasBeenValidated")
|
||||
.isEqualTo(qInstance);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,75 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public class S3UtilsTest extends BaseS3Test
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testListObjectsInBucketAtPath()
|
||||
{
|
||||
S3Utils s3Utils = getS3Utils();
|
||||
assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/", false).size(), "Expected # of s3 objects without subfolders");
|
||||
assertEquals(3, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/", true).size(), "Expected # of s3 objects with subfolders");
|
||||
assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "/" + TEST_FOLDER + "/", false).size(), "With leading slash");
|
||||
assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "/" + TEST_FOLDER, false).size(), "Without trailing slash");
|
||||
assertEquals(2, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "//" + TEST_FOLDER + "//", false).size(), "With multiple leading and trailing slashes");
|
||||
assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, false).size(), "Just in the subfolder non-recursive");
|
||||
assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, true).size(), "Just in the subfolder recursive");
|
||||
assertEquals(1, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, TEST_FOLDER + "//" + SUB_FOLDER, true).size(), "Just in the subfolder recursive");
|
||||
assertEquals(0, s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "not-a-real-path/", true).size(), "In a non-existing folder");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testGetObjectAsInputStream() throws IOException
|
||||
{
|
||||
S3Utils s3Utils = getS3Utils();
|
||||
S3ObjectSummary s3ObjectSummary = s3Utils.listObjectsInBucketAtPath(BUCKET_NAME, "test-files", true).get(0);
|
||||
InputStream inputStream = s3Utils.getObjectAsInputStream(s3ObjectSummary);
|
||||
String csvFromS3 = IOUtils.toString(inputStream);
|
||||
|
||||
// todo - should check the filename somewhere, right?
|
||||
assertEquals(getCSVData1(), csvFromS3, "File from S3 should match expected content");
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user