QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-17 20:50:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Initial add of sftp filesystem module

Browse Source
This commit is contained in:
darin.kelkhoff
2025-02-14 20:26:44 -06:00
parent 5a7199495d
commit b93114a9ba
23 changed files with 2176 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
qqq-backend-module-filesystem/pom.xml
View File

@ -50,6 +50,17 @@
<artifactId>aws-java-sdk-s3</artifactId> <artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.261</version> <version>1.12.261</version>
</dependency> </dependency>
<dependency>
<groupId>org.apache.sshd</groupId>
<artifactId>sshd-sftp</artifactId>
<version>2.14.0</version>
</dependency>
<dependency>
<groupId>org.apache.sshd</groupId>
<artifactId>sshd-sftp</artifactId>
<version>2.14.0</version>
</dependency>
<dependency> <dependency>
<groupId>cloud.localstack</groupId> <groupId>cloud.localstack</groupId>
<artifactId>localstack-utils</artifactId> <artifactId>localstack-utils</artifactId>
@ -57,6 +68,19 @@
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>1.15.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>5.7.0</version>
<scope>test</scope>
</dependency>
<!-- Common deps for all qqq modules --> <!-- Common deps for all qqq modules -->
<dependency> <dependency>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>

170
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/SFTPBackendModule.java Normal file
View File

@ -0,0 +1,170 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp;
import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.AbstractSFTPAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPCountAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPDeleteAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPInsertAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPQueryAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPStorageAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPUpdateAction;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails;
/*******************************************************************************
** QQQ Backend module for working with SFTP filesystems (as a client)
*******************************************************************************/
public class SFTPBackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface
{
public static final String BACKEND_TYPE = "sftp";
static
{
QBackendModuleDispatcher.registerBackendModule(new SFTPBackendModule());
}
/*******************************************************************************
** For filesystem backends, get the module-specific action base-class, that helps
** with functions like listing and deleting files.
*******************************************************************************/
@Override
public AbstractBaseFilesystemAction<SFTPDirEntryWithPath> getActionBase()
{
return (new AbstractSFTPAction());
}
/*******************************************************************************
** Method where a backend module must be able to provide its type (name).
*******************************************************************************/
@Override
public String getBackendType()
{
return (BACKEND_TYPE);
}
/*******************************************************************************
** Method to identify the class used for backend meta data for this module.
*******************************************************************************/
@Override
public Class<? extends QBackendMetaData> getBackendMetaDataClass()
{
return (SFTPBackendMetaData.class);
}
/*******************************************************************************
** Method to identify the class used for table-backend details for this module.
*******************************************************************************/
@Override
public Class<? extends QTableBackendDetails> getTableBackendDetailsClass()
{
return (SFTPTableBackendDetails.class);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QueryInterface getQueryInterface()
{
return new SFTPQueryAction();
}
/***************************************************************************
**
***************************************************************************/
@Override
public CountInterface getCountInterface()
{
return new SFTPCountAction();
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public InsertInterface getInsertInterface()
{
return (new SFTPInsertAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public UpdateInterface getUpdateInterface()
{
return (new SFTPUpdateAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public DeleteInterface getDeleteInterface()
{
return (new SFTPDeleteAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QStorageInterface getStorageInterface()
{
return new SFTPStorageAction();
}
}

298
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java Normal file
View File

@ -0,0 +1,298 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting;
import org.apache.sshd.client.SshClient;
import org.apache.sshd.client.session.ClientSession;
import org.apache.sshd.sftp.client.SftpClient;
import org.apache.sshd.sftp.client.SftpClientFactory;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** Base class for all SFTP filesystem actions
*******************************************************************************/
public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntryWithPath>
{
private static final QLogger LOG = QLogger.getLogger(AbstractSFTPAction.class);
private SshClient sshClient;
private ClientSession clientSession;
private SftpClient sftpClient;
/*******************************************************************************
** Set up the sftp utils object to be used for this action.
*******************************************************************************/
@Override
public void preAction(QBackendMetaData backendMetaData) throws QException
{
super.preAction(backendMetaData);
if(sftpClient != null)
{
LOG.debug("sftpClient object is already set - not re-setting it.");
return;
}
try
{
SFTPBackendMetaData sftpBackendMetaData = getBackendMetaData(SFTPBackendMetaData.class, backendMetaData);
this.sshClient = SshClient.setUpDefaultClient();
sshClient.start();
String username = sftpBackendMetaData.getUsername();
String password = sftpBackendMetaData.getPassword();
String hostName = sftpBackendMetaData.getHostName();
Integer port = sftpBackendMetaData.getPort();
if(backendMetaData.getUsesVariants())
{
QRecord variantRecord = getVariantRecord(backendMetaData);
LOG.debug("Getting SFTP connection credentials from variant record",
logPair("tableName", backendMetaData.getBackendVariantsConfig().getOptionsTableName()),
logPair("id", variantRecord.getValue("id")),
logPair("name", variantRecord.getRecordLabel()));
Map<BackendVariantSetting, String> fieldNameMap = backendMetaData.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap();
if(fieldNameMap.containsKey(SFTPBackendVariantSetting.USERNAME))
{
username = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.USERNAME));
}
if(fieldNameMap.containsKey(SFTPBackendVariantSetting.PASSWORD))
{
password = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.PASSWORD));
}
if(fieldNameMap.containsKey(SFTPBackendVariantSetting.HOSTNAME))
{
hostName = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.HOSTNAME));
}
if(fieldNameMap.containsKey(SFTPBackendVariantSetting.PORT))
{
port = variantRecord.getValueInteger(fieldNameMap.get(SFTPBackendVariantSetting.PORT));
}
}
this.clientSession = sshClient.connect(username, hostName, port).verify().getSession();
clientSession.addPasswordIdentity(password);
clientSession.auth().verify();
this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession);
}
catch(IOException e)
{
throw (new QException("Error setting up SFTP connection", e));
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public Long getFileSize(SFTPDirEntryWithPath sftpDirEntryWithPath)
{
try
{
return sftpDirEntryWithPath.dirEntry().getAttributes().getSize();
}
catch(Exception e)
{
return (null);
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public Instant getFileCreateDate(SFTPDirEntryWithPath sftpDirEntryWithPath)
{
try
{
return sftpDirEntryWithPath.dirEntry().getAttributes().getCreateTime().toInstant();
}
catch(Exception e)
{
return (null);
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public Instant getFileModifyDate(SFTPDirEntryWithPath sftpDirEntryWithPath)
{
try
{
return sftpDirEntryWithPath.dirEntry().getAttributes().getModifyTime().toInstant();
}
catch(Exception e)
{
return (null);
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public List<SFTPDirEntryWithPath> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
{
try
{
String fullPath = getFullBasePath(table, backendBase);
List<SFTPDirEntryWithPath> rs = new ArrayList<>();
for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath))
{
if(".".equals(dirEntry.getFilename()) || "..".equals(dirEntry.getFilename()))
{
continue;
}
if(dirEntry.getAttributes().isDirectory())
{
// todo - recursive??
continue;
}
// todo filter/glob
// todo skip
// todo limit
// todo order by
rs.add(new SFTPDirEntryWithPath(fullPath, dirEntry));
}
return (rs);
}
catch(Exception e)
{
throw (new QException("Error listing files", e));
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public InputStream readFile(SFTPDirEntryWithPath dirEntry) throws IOException
{
return (sftpClient.read(getFullPathForFile(dirEntry)));
}
/***************************************************************************
**
***************************************************************************/
@Override
public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException
{
sftpClient.put(new ByteArrayInputStream(contents), path);
}
/***************************************************************************
**
***************************************************************************/
@Override
public String getFullPathForFile(SFTPDirEntryWithPath dirEntry)
{
return (dirEntry.path() + "/" + dirEntry.dirEntry().getFilename());
}
/***************************************************************************
**
***************************************************************************/
@Override
public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
{
throw (new QRuntimeException("Not yet implemented"));
}
/***************************************************************************
**
***************************************************************************/
@Override
public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException
{
throw (new QRuntimeException("Not yet implemented"));
}
/***************************************************************************
*
***************************************************************************/
protected SftpClient getSftpClient(QBackendMetaData backend) throws QException
{
if(sftpClient == null)
{
preAction(backend);
}
return (sftpClient);
}
}

45
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java Normal file
View File

@ -0,0 +1,45 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPCountAction extends AbstractSFTPAction implements CountInterface
{
/*******************************************************************************
**
*******************************************************************************/
public CountOutput execute(CountInput countInput) throws QException
{
return (executeCount(countInput));
}
}

60
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java Normal file
View File

@ -0,0 +1,60 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPDeleteAction implements DeleteInterface
{
/*******************************************************************************
**
*******************************************************************************/
public DeleteOutput execute(DeleteInput deleteInput) throws QException
{
throw new NotImplementedException("SFTP delete not implemented");
/*
try
{
DeleteResult rs = new DeleteResult();
QTableMetaData table = deleteRequest.getTable();
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing delete: " + e.getMessage(), e);
}
*/
}
}

45
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java Normal file
View File

@ -0,0 +1,45 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPInsertAction extends AbstractSFTPAction implements InsertInterface
{
/*******************************************************************************
**
*******************************************************************************/
public InsertOutput execute(InsertInput insertInput) throws QException
{
return (super.executeInsert(insertInput));
}
}

45
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java Normal file
View File

@ -0,0 +1,45 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPQueryAction extends AbstractSFTPAction implements QueryInterface
{
/*******************************************************************************
**
*******************************************************************************/
public QueryOutput execute(QueryInput queryInput) throws QException
{
return (super.executeQuery(queryInput));
}
}

157
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java Normal file
View File

@ -0,0 +1,157 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import java.io.File;
import java.io.InputStream;
import java.io.OutputStream;
import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.sftp.utils.SFTPOutputStream;
import org.apache.sshd.sftp.client.SftpClient;
/*******************************************************************************
** (mass, streamed) storage action for sftp module
*******************************************************************************/
public class SFTPStorageAction extends AbstractSFTPAction implements QStorageInterface
{
/*******************************************************************************
** create an output stream in the storage backend - that can be written to,
** for the purpose of inserting or writing a file into storage.
*******************************************************************************/
@Override
public OutputStream createOutputStream(StorageInput storageInput) throws QException
{
try
{
SFTPBackendMetaData backend = (SFTPBackendMetaData) storageInput.getBackend();
preAction(backend);
SftpClient sftpClient = getSftpClient(backend);
SFTPOutputStream sftpOutputStream = new SFTPOutputStream(sftpClient, getFullPath(storageInput));
return (sftpOutputStream);
}
catch(Exception e)
{
throw (new QException("Exception creating sftp output stream for file", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
private String getFullPath(StorageInput storageInput) throws QException
{
QTableMetaData table = storageInput.getTable();
QBackendMetaData backend = storageInput.getBackend();
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + storageInput.getReference());
/////////////////////////////////////////////////////////////
// s3 seems to do better w/o leading slashes, so, strip... //
/////////////////////////////////////////////////////////////
if(fullPath.startsWith("/"))
{
fullPath = fullPath.substring(1);
}
return fullPath;
}
/*******************************************************************************
** create an input stream in the storage backend - that can be read from,
** for the purpose of getting or reading a file from storage.
*******************************************************************************/
@Override
public InputStream getInputStream(StorageInput storageInput) throws QException
{
try
{
SFTPBackendMetaData backend = (SFTPBackendMetaData) storageInput.getBackend();
preAction(backend);
SftpClient sftpClient = getSftpClient(backend);
InputStream inputStream = sftpClient.read(getFullPath(storageInput));
return (inputStream);
}
catch(Exception e)
{
throw (new QException("Exception getting sftp input stream for file.", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String getDownloadURL(StorageInput storageInput) throws QException
{
try
{
throw new QRuntimeException("Not implemented");
//S3BackendMetaData backend = (S3BackendMetaData) storageInput.getBackend();
//preAction(backend);
//
//AmazonS3 amazonS3 = getS3Utils().getAmazonS3();
//String fullPath = getFullPath(storageInput);
//return (amazonS3.getUrl(backend.getBucketName(), fullPath).toString());
}
catch(Exception e)
{
throw (new QException("Exception getting the sftp download URL.", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void makePublic(StorageInput storageInput) throws QException
{
try
{
throw new QRuntimeException("Not implemented");
}
catch(Exception e)
{
throw (new QException("Exception making sftp file publicly available.", e));
}
}
}

62
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java Normal file
View File

@ -0,0 +1,62 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPUpdateAction implements UpdateInterface
{
/*******************************************************************************
**
*******************************************************************************/
public UpdateOutput execute(UpdateInput updateInput) throws QException
{
throw new NotImplementedException("SFTP update not implemented");
/*
try
{
UpdateResult rs = new UpdateResult();
QTableMetaData table = updateRequest.getTable();
List<QRecord> records = new ArrayList<>();
rs.setRecords(records);
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing update: " + e.getMessage(), e);
}
*/
}
}

33
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java Normal file
View File

@ -0,0 +1,33 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.model;
import org.apache.sshd.sftp.client.SftpClient;
/*******************************************************************************
**
*******************************************************************************/
public record SFTPDirEntryWithPath(String path, SftpClient.DirEntry dirEntry)
{
}

198
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java Normal file
View File

@ -0,0 +1,198 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule;
/*******************************************************************************
** SFTP backend meta data.
*******************************************************************************/
public class SFTPBackendMetaData extends AbstractFilesystemBackendMetaData
{
private String username;
private String password;
private String hostName;
private Integer port;
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public SFTPBackendMetaData()
{
super();
setBackendType(SFTPBackendModule.class);
}
/*******************************************************************************
** Fluent setter for basePath
**
*******************************************************************************/
public SFTPBackendMetaData withBasePath(String basePath)
{
setBasePath(basePath);
return (this);
}
/*******************************************************************************
** Fluent setter for name
**
*******************************************************************************/
public SFTPBackendMetaData withName(String name)
{
setName(name);
return this;
}
/*******************************************************************************
** Getter for username
*******************************************************************************/
public String getUsername()
{
return (this.username);
}
/*******************************************************************************
** Setter for username
*******************************************************************************/
public void setUsername(String username)
{
this.username = username;
}
/*******************************************************************************
** Fluent setter for username
*******************************************************************************/
public SFTPBackendMetaData withUsername(String username)
{
this.username = username;
return (this);
}
/*******************************************************************************
** Getter for password
*******************************************************************************/
public String getPassword()
{
return (this.password);
}
/*******************************************************************************
** Setter for password
*******************************************************************************/
public void setPassword(String password)
{
this.password = password;
}
/*******************************************************************************
** Fluent setter for password
*******************************************************************************/
public SFTPBackendMetaData withPassword(String password)
{
this.password = password;
return (this);
}
/*******************************************************************************
** Getter for hostName
*******************************************************************************/
public String getHostName()
{
return (this.hostName);
}
/*******************************************************************************
** Setter for hostName
*******************************************************************************/
public void setHostName(String hostName)
{
this.hostName = hostName;
}
/*******************************************************************************
** Fluent setter for hostName
*******************************************************************************/
public SFTPBackendMetaData withHostName(String hostName)
{
this.hostName = hostName;
return (this);
}
/*******************************************************************************
** Getter for port
*******************************************************************************/
public Integer getPort()
{
return (this.port);
}
/*******************************************************************************
** Setter for port
*******************************************************************************/
public void setPort(Integer port)
{
this.port = port;
}
/*******************************************************************************
** Fluent setter for port
*******************************************************************************/
public SFTPBackendMetaData withPort(Integer port)
{
this.port = port;
return (this);
}
}

38
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java Normal file
View File

@ -0,0 +1,38 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
/*******************************************************************************
**
*******************************************************************************/
public enum SFTPBackendVariantSetting implements BackendVariantSetting
{
USERNAME,
PASSWORD,
HOSTNAME,
PORT,
BASE_PATH
}

45
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java Normal file
View File

@ -0,0 +1,45 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule;
/*******************************************************************************
** SFTP specific Extension of QTableBackendDetails
*******************************************************************************/
public class SFTPTableBackendDetails extends AbstractFilesystemTableBackendDetails
{
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public SFTPTableBackendDetails()
{
super();
setBackendType(SFTPBackendModule.class);
}
}

179
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java Normal file
View File

@ -0,0 +1,179 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.utils;
import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import org.apache.sshd.sftp.client.SftpClient;
import org.jetbrains.annotations.NotNull;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPOutputStream extends PipedOutputStream
{
private static final QLogger LOG = QLogger.getLogger(SFTPOutputStream.class);
private final SftpClient sftpClient;
private final PipedInputStream pipedInputStream;
private final Future<?> putFuture;
private AtomicBoolean started = new AtomicBoolean(false);
private AtomicReference<Exception> putException = new AtomicReference<>(null);
/***************************************************************************
**
***************************************************************************/
public SFTPOutputStream(SftpClient sftpClient, String path) throws IOException
{
pipedInputStream = new PipedInputStream(this, 32 * 1024);
this.sftpClient = sftpClient;
putFuture = Executors.newSingleThreadExecutor().submit(() ->
{
try
{
started.set(true);
sftpClient.put(pipedInputStream, path);
}
catch(Exception e)
{
putException.set(e);
LOG.error("Error putting SFTP output stream", e);
try
{
pipedInputStream.close();
}
catch(IOException ex)
{
LOG.error("Secondary error closing pipedInputStream after sftp put error", e);
}
throw new RuntimeException(e);
}
});
}
/***************************************************************************
**
***************************************************************************/
@Override
public void write(@NotNull byte[] b) throws IOException
{
try
{
super.write(b);
}
catch(IOException e)
{
if(putException.get() != null)
{
throw new IOException("Error performing SFTP put", putException.get());
}
throw new IOException("Error writing to SFTP output stream", e);
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public void close() throws IOException
{
try
{
/////////////////////////////////////////////////////////////////////////////////////////////
// don't try to close anything until we know that the sftpClient.put call's thread //
// has tried to start (otherwise, race condition could cause us to close things too early) //
/////////////////////////////////////////////////////////////////////////////////////////////
int sleepLoops = 0;
while(!started.get() && sleepLoops++ <= 30)
{
SleepUtils.sleep(1, TimeUnit.SECONDS);
}
///////////////////////////////////////////////////////////////////////////////////////
// closing the pipedOutputStream (super) causes things to flush and complete the put //
///////////////////////////////////////////////////////////////////////////////////////
super.close();
////////////////////////////////
// wait for the put to finish //
////////////////////////////////
putFuture.get(60 - sleepLoops, TimeUnit.SECONDS);
///////////////////////////////////////////////////////////////////////////////
// in case the put-future never did start, throw explicitly mentioning that. //
///////////////////////////////////////////////////////////////////////////////
if(sleepLoops >= 30)
{
throw (new Exception("future to can sftpClient.put() was never started."));
}
}
catch(ExecutionException ee)
{
throw new IOException("Error performing SFTP put", ee);
}
catch(Exception e)
{
if(putException.get() != null)
{
throw new IOException("Error performing SFTP put", putException.get());
}
throw new IOException("Error closing SFTP output stream", e);
}
finally
{
try
{
sftpClient.close();
}
catch(IOException e)
{
LOG.error("Error closing SFTP client", e);
}
}
}
}

140
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.filesystem;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@ -37,12 +38,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig;
import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule; import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.FilesystemTableMetaDataBuilder;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
@ -52,6 +55,10 @@ import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fil
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
@ -60,20 +67,26 @@ import org.apache.commons.io.FileUtils;
*******************************************************************************/ *******************************************************************************/
public class TestUtils public class TestUtils
{ {
public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
public static final String BACKEND_NAME_S3 = "s3"; public static final String BACKEND_NAME_S3 = "s3";
public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix";
public static final String BACKEND_NAME_MOCK = "mock"; public static final String BACKEND_NAME_SFTP = "sftp";
public static final String BACKEND_NAME_MEMORY = "memory"; public static final String BACKEND_NAME_SFTP_WITH_VARIANTS = "sftpWithVariants";
public static final String BACKEND_NAME_MOCK = "mock";
public static final String BACKEND_NAME_MEMORY = "memory";
public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json"; public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json";
public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv"; public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv";
public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob"; public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob";
public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive"; public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive";
public static final String TABLE_NAME_PERSON_S3 = "person-s3"; public static final String TABLE_NAME_PERSON_S3 = "person-s3";
public static final String TABLE_NAME_PERSON_SFTP = "person-sftp";
public static final String TABLE_NAME_BLOB_S3 = "s3-blob"; public static final String TABLE_NAME_BLOB_S3 = "s3-blob";
public static final String TABLE_NAME_PERSON_MOCK = "person-mock"; public static final String TABLE_NAME_PERSON_MOCK = "person-mock";
public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix"; public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix";
public static final String TABLE_NAME_SFTP_FILE = "sftp-file";
public static final String TABLE_NAME_SFTP_FILE_VARIANTS = "sftp-file-with-variants";
public static final String TABLE_NAME_VARIANT_OPTIONS = "variant-options-table";
public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed"; public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed";
public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter"; public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter";
@ -148,6 +161,7 @@ public class TestUtils
qInstance.addBackend(defineS3Backend()); qInstance.addBackend(defineS3Backend());
qInstance.addBackend(defineS3BackendSansPrefix()); qInstance.addBackend(defineS3BackendSansPrefix());
qInstance.addTable(defineS3CSVPersonTable()); qInstance.addTable(defineS3CSVPersonTable());
qInstance.addTable(defineSFTPCSVPersonTable());
qInstance.addTable(defineS3BlobTable()); qInstance.addTable(defineS3BlobTable());
qInstance.addTable(defineS3BlobSansPrefixTable()); qInstance.addTable(defineS3BlobSansPrefixTable());
qInstance.addBackend(defineMockBackend()); qInstance.addBackend(defineMockBackend());
@ -155,6 +169,15 @@ public class TestUtils
qInstance.addTable(defineMockPersonTable()); qInstance.addTable(defineMockPersonTable());
qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess()); qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
QBackendMetaData sftpBackend = defineSFTPBackend();
qInstance.addBackend(sftpBackend);
qInstance.addTable(defineSFTPFileTable(sftpBackend));
QBackendMetaData sftpBackendWithVariants = defineSFTPBackendWithVariants();
qInstance.addBackend(sftpBackendWithVariants);
qInstance.addTable(defineSFTPFileTableWithVariants(sftpBackendWithVariants));
qInstance.addTable(defineVariantOptionsTable());
definePersonCsvImporter(qInstance); definePersonCsvImporter(qInstance);
return (qInstance); return (qInstance);
@ -162,6 +185,21 @@ public class TestUtils
/***************************************************************************
**
***************************************************************************/
private static QTableMetaData defineVariantOptionsTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_VARIANT_OPTIONS)
.withBackendName(defineMemoryBackend().getName())
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("basePath", QFieldType.STRING));
}
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/
@ -379,6 +417,25 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineSFTPCSVPersonTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_PERSON_SFTP)
.withLabel("Person SFTP Table")
.withBackendName(BACKEND_NAME_SFTP)
.withPrimaryKeyField("id")
.withFields(defineCommonPersonTableFields())
.withBackendDetails(new SFTPTableBackendDetails()
.withRecordFormat(RecordFormat.CSV)
.withCardinality(Cardinality.MANY)
);
}
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/
@ -463,4 +520,77 @@ public class TestUtils
MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule(); MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule();
return (mockAuthenticationModule.createSession(defineInstance(), null)); return (mockAuthenticationModule.createSession(defineInstance(), null));
} }
/***************************************************************************
**
***************************************************************************/
public static QTableMetaData defineSFTPFileTable(QBackendMetaData sftpBackend)
{
return new FilesystemTableMetaDataBuilder()
.withBasePath(BaseSFTPTest.TABLE_FOLDER)
.withBackend(sftpBackend)
.withName(TABLE_NAME_SFTP_FILE)
.buildStandardCardinalityOneTable()
.withLabel("SFTP Files");
}
/***************************************************************************
**
***************************************************************************/
private static QBackendMetaData defineSFTPBackend()
{
return (new SFTPBackendMetaData()
.withUsername(BaseSFTPTest.USERNAME)
.withPassword(BaseSFTPTest.PASSWORD)
.withHostName(BaseSFTPTest.HOST_NAME)
.withPort(BaseSFTPTest.getCurrentPort())
.withBasePath(BaseSFTPTest.BACKEND_FOLDER)
.withName(BACKEND_NAME_SFTP));
}
/***************************************************************************
**
***************************************************************************/
public static QTableMetaData defineSFTPFileTableWithVariants(QBackendMetaData sftpBackend)
{
return new FilesystemTableMetaDataBuilder()
.withBasePath(BaseSFTPTest.TABLE_FOLDER)
.withBackend(sftpBackend)
.withName(TABLE_NAME_SFTP_FILE_VARIANTS)
.buildStandardCardinalityOneTable()
.withLabel("SFTP Files");
}
/***************************************************************************
**
***************************************************************************/
private static QBackendMetaData defineSFTPBackendWithVariants()
{
return (new SFTPBackendMetaData()
.withUsername(BaseSFTPTest.USERNAME)
.withPassword(BaseSFTPTest.PASSWORD)
.withHostName(BaseSFTPTest.HOST_NAME)
.withPort(BaseSFTPTest.getCurrentPort())
////////////////////////////////////
// only get basePath from variant //
////////////////////////////////////
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withOptionsTableName(TABLE_NAME_VARIANT_OPTIONS)
.withVariantTypeKey(TABLE_NAME_VARIANT_OPTIONS)
.withBackendSettingSourceFieldNameMap(Map.of(
SFTPBackendVariantSetting.BASE_PATH, "basePath"
))
)
.withName(BACKEND_NAME_SFTP_WITH_VARIANTS));
}
} }

139
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java Normal file
View File

@ -0,0 +1,139 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp;
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.containers.Container;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.utility.MountableFile;
/*******************************************************************************
** Base class for tests that want to be able to work with sftp testcontainer
*******************************************************************************/
public class BaseSFTPTest extends BaseTest
{
public static final int PORT = 22;
public static final String USERNAME = "testuser";
public static final String PASSWORD = "testpass";
public static final String HOST_NAME = "localhost";
public static final String BACKEND_FOLDER = "upload";
public static final String TABLE_FOLDER = "files";
public static final String REMOTE_DIR = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER;
private static GenericContainer<?> sftpContainer;
private static Integer currentPort;
/***************************************************************************
**
***************************************************************************/
@BeforeAll
static void setUp() throws Exception
{
sftpContainer = new GenericContainer<>("atmoz/sftp:latest")
.withExposedPorts(PORT)
.withCommand(USERNAME + ":" + PASSWORD + ":1001");
sftpContainer.start();
for(int i = 0; i < 5; i++)
{
sftpContainer.copyFileToContainer(MountableFile.forClasspathResource("files/testfile.txt"), REMOTE_DIR + "/testfile-" + i + ".txt");
}
grantUploadFilesDirWritePermission();
currentPort = sftpContainer.getMappedPort(22);
}
/***************************************************************************
**
***************************************************************************/
@AfterAll
static void tearDown()
{
if(sftpContainer != null)
{
sftpContainer.stop();
}
}
/*******************************************************************************
** Getter for currentPort
**
*******************************************************************************/
public static Integer getCurrentPort()
{
return currentPort;
}
/***************************************************************************
**
***************************************************************************/
protected static void revokeUploadFilesDirWritePermission() throws Exception
{
setUploadFilesDirPermission("444");
}
/***************************************************************************
**
***************************************************************************/
protected static void grantUploadFilesDirWritePermission() throws Exception
{
setUploadFilesDirPermission("777");
}
/***************************************************************************
**
***************************************************************************/
private static void setUploadFilesDirPermission(String mode) throws Exception
{
sftpContainer.execInContainer("chmod", mode, "/home/testuser/upload/files");
}
/***************************************************************************
*
***************************************************************************/
protected void mkdirInSftpContainerUnderHomeTestuser(String path) throws Exception
{
Container.ExecResult mkdir = sftpContainer.execInContainer("mkdir", "-p", "/home/testuser/" + path);
System.out.println(mkdir.getExitCode());
}
}

64
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java Normal file
View File

@ -0,0 +1,64 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPCountActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCount1() throws QException
{
CountInput countInput = initCountRequest();
SFTPCountAction countAction = new SFTPCountAction();
CountOutput countOutput = countAction.execute(countInput);
Assertions.assertEquals(5, countOutput.getCount(), "Expected # of rows from unfiltered count");
}
/*******************************************************************************
**
*******************************************************************************/
private CountInput initCountRequest() throws QException
{
CountInput countInput = new CountInput();
countInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE);
return countInput;
}
}

48
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java Normal file
View File

@ -0,0 +1,48 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertThrows;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPDeleteActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void test() throws QException
{
assertThrows(NotImplementedException.class, () -> new SFTPDeleteAction().execute(new DeleteInput()));
}
}

120
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java Normal file
View File

@ -0,0 +1,120 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import java.io.IOException;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPInsertActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCardinalityOne() throws QException, IOException
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
));
SFTPInsertAction insertAction = new SFTPInsertAction();
InsertOutput insertOutput = insertAction.execute(insertInput);
assertThat(insertOutput.getRecords())
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseSFTPTest.BACKEND_FOLDER));
QRecord record = insertOutput.getRecords().get(0);
String fullPath = record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
assertThat(record.getErrors()).isNullOrEmpty();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCardinalityOnePermissionError() throws Exception
{
try
{
revokeUploadFilesDirWritePermission();
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
));
SFTPInsertAction insertAction = new SFTPInsertAction();
InsertOutput insertOutput = insertAction.execute(insertInput);
QRecord record = insertOutput.getRecords().get(0);
assertThat(record.getErrors()).isNotEmpty();
assertThat(record.getErrors().get(0).getMessage()).contains("Error writing file: Permission denied");
}
finally
{
grantUploadFilesDirWritePermission();
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCardinalityMany() throws QException, IOException
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_SFTP);
insertInput.setRecords(List.of(
new QRecord().withValue("id", "1").withValue("firstName", "Bob")
));
SFTPInsertAction insertAction = new SFTPInsertAction();
assertThatThrownBy(() -> insertAction.execute(insertInput))
.hasRootCauseInstanceOf(NotImplementedException.class);
}
}

105
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java Normal file
View File

@ -0,0 +1,105 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Unit test for SFTPQueryAction
*******************************************************************************/
class SFTPQueryActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQuery1() throws QException
{
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQueryVariantsTable() throws Exception
{
new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_VARIANT_OPTIONS).withRecords(List.of(
new QRecord().withValue("id", 1).withValue("basePath", BaseSFTPTest.BACKEND_FOLDER),
new QRecord().withValue("id", 2).withValue("basePath", "empty-folder"),
new QRecord().withValue("id", 3).withValue("basePath", "non-existing-path")
)));
mkdirInSftpContainerUnderHomeTestuser("empty-folder/files");
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS);
assertThatThrownBy(() -> new QueryAction().execute(queryInput))
.hasMessageContaining("Could not find Backend Variant information for Backend");
QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 1));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 2));
queryOutput = new QueryAction().execute(queryInput);
Assertions.assertEquals(0, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 3));
assertThatThrownBy(() -> new QueryAction().execute(queryInput))
.rootCause()
.hasMessageContaining("No such file");
// Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
}
/*******************************************************************************
**
*******************************************************************************/
private QueryInput initQueryRequest() throws QException
{
QueryInput queryInput = new QueryInput();
return queryInput;
}
}

115
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java Normal file
View File

@ -0,0 +1,115 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for FilesystemStorageAction
*******************************************************************************/
public class SFTPStorageActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testSmall() throws Exception
{
String data = "Hellooo, Storage.";
runTest(data);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testPermissionError() throws Exception
{
try
{
revokeUploadFilesDirWritePermission();
String data = "oops!";
assertThatThrownBy(() -> runTest(data))
.hasRootCauseInstanceOf(IOException.class)
.rootCause()
.hasMessageContaining("Permission denied");
}
finally
{
grantUploadFilesDirWritePermission();
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testLarge() throws Exception
{
String data = StringUtils.join("!", Collections.nCopies(5_000_000, "Hello"));
runTest(data);
}
/***************************************************************************
**
***************************************************************************/
private static void runTest(String data) throws QException, IOException
{
StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_SFTP_FILE).withReference("fromStorageAction.txt");
StorageAction storageAction = new StorageAction();
OutputStream outputStream = storageAction.createOutputStream(storageInput);
outputStream.write(data.getBytes(StandardCharsets.UTF_8));
outputStream.close();
InputStream inputStream = storageAction.getInputStream(storageInput);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
inputStream.transferTo(byteArrayOutputStream);
assertEquals(data.length(), byteArrayOutputStream.toString(StandardCharsets.UTF_8).length());
assertEquals(data, byteArrayOutputStream.toString(StandardCharsets.UTF_8));
}
}

48
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java Normal file
View File

@ -0,0 +1,48 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertThrows;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPUpdateActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void test() throws QException
{
assertThrows(NotImplementedException.class, () -> new SFTPUpdateAction().execute(new UpdateInput()));
}
}

3
qqq-backend-module-filesystem/src/test/resources/files/testfile.txt Normal file
View File

@ -0,0 +1,3 @@
This is a file.
It is a test.