diff --git a/qqq-backend-module-filesystem/pom.xml b/qqq-backend-module-filesystem/pom.xml index 32cc274f..9344db4a 100644 --- a/qqq-backend-module-filesystem/pom.xml +++ b/qqq-backend-module-filesystem/pom.xml @@ -50,6 +50,17 @@ aws-java-sdk-s3 1.12.261 + + org.apache.sshd + sshd-sftp + 2.14.0 + + + org.apache.sshd + sshd-sftp + 2.14.0 + + cloud.localstack localstack-utils @@ -57,6 +68,19 @@ test + + org.testcontainers + testcontainers + 1.15.3 + test + + + net.java.dev.jna + jna + 5.7.0 + test + + org.apache.maven.plugins diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/SFTPBackendModule.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/SFTPBackendModule.java new file mode 100644 index 00000000..17c447dd --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/SFTPBackendModule.java @@ -0,0 +1,170 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.AbstractSFTPAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPCountAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPDeleteAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPInsertAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPQueryAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPStorageAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPUpdateAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails; + + +/******************************************************************************* + ** QQQ Backend module for working with SFTP filesystems (as a client) + *******************************************************************************/ +public class SFTPBackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface +{ + public static final String BACKEND_TYPE = "sftp"; + + static + { + QBackendModuleDispatcher.registerBackendModule(new SFTPBackendModule()); + } + + /******************************************************************************* + ** For filesystem backends, get the module-specific action base-class, that helps + ** with functions like listing and deleting files. + *******************************************************************************/ + @Override + public AbstractBaseFilesystemAction getActionBase() + { + return (new AbstractSFTPAction()); + } + + + + /******************************************************************************* + ** Method where a backend module must be able to provide its type (name). + *******************************************************************************/ + @Override + public String getBackendType() + { + return (BACKEND_TYPE); + } + + + + /******************************************************************************* + ** Method to identify the class used for backend meta data for this module. + *******************************************************************************/ + @Override + public Class getBackendMetaDataClass() + { + return (SFTPBackendMetaData.class); + } + + + + /******************************************************************************* + ** Method to identify the class used for table-backend details for this module. + *******************************************************************************/ + @Override + public Class getTableBackendDetailsClass() + { + return (SFTPTableBackendDetails.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryInterface getQueryInterface() + { + return new SFTPQueryAction(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public CountInterface getCountInterface() + { + return new SFTPCountAction(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InsertInterface getInsertInterface() + { + return (new SFTPInsertAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public UpdateInterface getUpdateInterface() + { + return (new SFTPUpdateAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public DeleteInterface getDeleteInterface() + { + return (new SFTPDeleteAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QStorageInterface getStorageInterface() + { + return new SFTPStorageAction(); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java new file mode 100644 index 00000000..80db887e --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -0,0 +1,298 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; +import org.apache.sshd.client.SshClient; +import org.apache.sshd.client.session.ClientSession; +import org.apache.sshd.sftp.client.SftpClient; +import org.apache.sshd.sftp.client.SftpClientFactory; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Base class for all SFTP filesystem actions + *******************************************************************************/ +public class AbstractSFTPAction extends AbstractBaseFilesystemAction +{ + private static final QLogger LOG = QLogger.getLogger(AbstractSFTPAction.class); + + private SshClient sshClient; + private ClientSession clientSession; + private SftpClient sftpClient; + + + + /******************************************************************************* + ** Set up the sftp utils object to be used for this action. + *******************************************************************************/ + @Override + public void preAction(QBackendMetaData backendMetaData) throws QException + { + super.preAction(backendMetaData); + + if(sftpClient != null) + { + LOG.debug("sftpClient object is already set - not re-setting it."); + return; + } + + try + { + SFTPBackendMetaData sftpBackendMetaData = getBackendMetaData(SFTPBackendMetaData.class, backendMetaData); + + this.sshClient = SshClient.setUpDefaultClient(); + sshClient.start(); + + String username = sftpBackendMetaData.getUsername(); + String password = sftpBackendMetaData.getPassword(); + String hostName = sftpBackendMetaData.getHostName(); + Integer port = sftpBackendMetaData.getPort(); + + if(backendMetaData.getUsesVariants()) + { + QRecord variantRecord = getVariantRecord(backendMetaData); + LOG.debug("Getting SFTP connection credentials from variant record", + logPair("tableName", backendMetaData.getBackendVariantsConfig().getOptionsTableName()), + logPair("id", variantRecord.getValue("id")), + logPair("name", variantRecord.getRecordLabel())); + Map fieldNameMap = backendMetaData.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap(); + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.USERNAME)) + { + username = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.USERNAME)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.PASSWORD)) + { + password = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.PASSWORD)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.HOSTNAME)) + { + hostName = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.HOSTNAME)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.PORT)) + { + port = variantRecord.getValueInteger(fieldNameMap.get(SFTPBackendVariantSetting.PORT)); + } + } + + this.clientSession = sshClient.connect(username, hostName, port).verify().getSession(); + clientSession.addPasswordIdentity(password); + clientSession.auth().verify(); + + this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession); + } + catch(IOException e) + { + throw (new QException("Error setting up SFTP connection", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Long getFileSize(SFTPDirEntryWithPath sftpDirEntryWithPath) + { + try + { + return sftpDirEntryWithPath.dirEntry().getAttributes().getSize(); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileCreateDate(SFTPDirEntryWithPath sftpDirEntryWithPath) + { + try + { + return sftpDirEntryWithPath.dirEntry().getAttributes().getCreateTime().toInstant(); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileModifyDate(SFTPDirEntryWithPath sftpDirEntryWithPath) + { + try + { + return sftpDirEntryWithPath.dirEntry().getAttributes().getModifyTime().toInstant(); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException + { + try + { + String fullPath = getFullBasePath(table, backendBase); + List rs = new ArrayList<>(); + + for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath)) + { + if(".".equals(dirEntry.getFilename()) || "..".equals(dirEntry.getFilename())) + { + continue; + } + + if(dirEntry.getAttributes().isDirectory()) + { + // todo - recursive?? + continue; + } + + // todo filter/glob + // todo skip + // todo limit + // todo order by + rs.add(new SFTPDirEntryWithPath(fullPath, dirEntry)); + } + + return (rs); + } + catch(Exception e) + { + throw (new QException("Error listing files", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public InputStream readFile(SFTPDirEntryWithPath dirEntry) throws IOException + { + return (sftpClient.read(getFullPathForFile(dirEntry))); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException + { + sftpClient.put(new ByteArrayInputStream(contents), path); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getFullPathForFile(SFTPDirEntryWithPath dirEntry) + { + return (dirEntry.path() + "/" + dirEntry.dirEntry().getFilename()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + { + throw (new QRuntimeException("Not yet implemented")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException + { + throw (new QRuntimeException("Not yet implemented")); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected SftpClient getSftpClient(QBackendMetaData backend) throws QException + { + if(sftpClient == null) + { + preAction(backend); + } + + return (sftpClient); + } +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java new file mode 100644 index 00000000..f882662c --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPCountAction extends AbstractSFTPAction implements CountInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public CountOutput execute(CountInput countInput) throws QException + { + return (executeCount(countInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java new file mode 100644 index 00000000..365e3ec7 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java @@ -0,0 +1,60 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPDeleteAction implements DeleteInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public DeleteOutput execute(DeleteInput deleteInput) throws QException + { + throw new NotImplementedException("SFTP delete not implemented"); + /* + try + { + DeleteResult rs = new DeleteResult(); + QTableMetaData table = deleteRequest.getTable(); + + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing delete: " + e.getMessage(), e); + } + */ + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java new file mode 100644 index 00000000..aa1c81dc --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPInsertAction extends AbstractSFTPAction implements InsertInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public InsertOutput execute(InsertInput insertInput) throws QException + { + return (super.executeInsert(insertInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java new file mode 100644 index 00000000..5759c7ba --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPQueryAction extends AbstractSFTPAction implements QueryInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public QueryOutput execute(QueryInput queryInput) throws QException + { + return (super.executeQuery(queryInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java new file mode 100644 index 00000000..ddcb40f1 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java @@ -0,0 +1,157 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.utils.SFTPOutputStream; +import org.apache.sshd.sftp.client.SftpClient; + + +/******************************************************************************* + ** (mass, streamed) storage action for sftp module + *******************************************************************************/ +public class SFTPStorageAction extends AbstractSFTPAction implements QStorageInterface +{ + + /******************************************************************************* + ** create an output stream in the storage backend - that can be written to, + ** for the purpose of inserting or writing a file into storage. + *******************************************************************************/ + @Override + public OutputStream createOutputStream(StorageInput storageInput) throws QException + { + try + { + SFTPBackendMetaData backend = (SFTPBackendMetaData) storageInput.getBackend(); + preAction(backend); + + SftpClient sftpClient = getSftpClient(backend); + + SFTPOutputStream sftpOutputStream = new SFTPOutputStream(sftpClient, getFullPath(storageInput)); + return (sftpOutputStream); + } + catch(Exception e) + { + throw (new QException("Exception creating sftp output stream for file", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getFullPath(StorageInput storageInput) throws QException + { + QTableMetaData table = storageInput.getTable(); + QBackendMetaData backend = storageInput.getBackend(); + String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + storageInput.getReference()); + + ///////////////////////////////////////////////////////////// + // s3 seems to do better w/o leading slashes, so, strip... // + ///////////////////////////////////////////////////////////// + if(fullPath.startsWith("/")) + { + fullPath = fullPath.substring(1); + } + + return fullPath; + } + + + + /******************************************************************************* + ** create an input stream in the storage backend - that can be read from, + ** for the purpose of getting or reading a file from storage. + *******************************************************************************/ + @Override + public InputStream getInputStream(StorageInput storageInput) throws QException + { + try + { + SFTPBackendMetaData backend = (SFTPBackendMetaData) storageInput.getBackend(); + preAction(backend); + + SftpClient sftpClient = getSftpClient(backend); + InputStream inputStream = sftpClient.read(getFullPath(storageInput)); + + return (inputStream); + } + catch(Exception e) + { + throw (new QException("Exception getting sftp input stream for file.", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDownloadURL(StorageInput storageInput) throws QException + { + try + { + throw new QRuntimeException("Not implemented"); + //S3BackendMetaData backend = (S3BackendMetaData) storageInput.getBackend(); + //preAction(backend); + // + //AmazonS3 amazonS3 = getS3Utils().getAmazonS3(); + //String fullPath = getFullPath(storageInput); + //return (amazonS3.getUrl(backend.getBucketName(), fullPath).toString()); + } + catch(Exception e) + { + throw (new QException("Exception getting the sftp download URL.", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void makePublic(StorageInput storageInput) throws QException + { + try + { + throw new QRuntimeException("Not implemented"); + } + catch(Exception e) + { + throw (new QException("Exception making sftp file publicly available.", e)); + } + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java new file mode 100644 index 00000000..93b31a69 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPUpdateAction implements UpdateInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public UpdateOutput execute(UpdateInput updateInput) throws QException + { + throw new NotImplementedException("SFTP update not implemented"); + /* + try + { + UpdateResult rs = new UpdateResult(); + QTableMetaData table = updateRequest.getTable(); + + List records = new ArrayList<>(); + rs.setRecords(records); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing update: " + e.getMessage(), e); + } + */ + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java new file mode 100644 index 00000000..06cae267 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java @@ -0,0 +1,33 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model; + + +import org.apache.sshd.sftp.client.SftpClient; + + +/******************************************************************************* + ** + *******************************************************************************/ +public record SFTPDirEntryWithPath(String path, SftpClient.DirEntry dirEntry) +{ +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java new file mode 100644 index 00000000..c423f99e --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java @@ -0,0 +1,198 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule; + + +/******************************************************************************* + ** SFTP backend meta data. + *******************************************************************************/ +public class SFTPBackendMetaData extends AbstractFilesystemBackendMetaData +{ + private String username; + private String password; + private String hostName; + private Integer port; + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public SFTPBackendMetaData() + { + super(); + setBackendType(SFTPBackendModule.class); + } + + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + public SFTPBackendMetaData withBasePath(String basePath) + { + setBasePath(basePath); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public SFTPBackendMetaData withName(String name) + { + setName(name); + return this; + } + + + + /******************************************************************************* + ** Getter for username + *******************************************************************************/ + public String getUsername() + { + return (this.username); + } + + + + /******************************************************************************* + ** Setter for username + *******************************************************************************/ + public void setUsername(String username) + { + this.username = username; + } + + + + /******************************************************************************* + ** Fluent setter for username + *******************************************************************************/ + public SFTPBackendMetaData withUsername(String username) + { + this.username = username; + return (this); + } + + + + /******************************************************************************* + ** Getter for password + *******************************************************************************/ + public String getPassword() + { + return (this.password); + } + + + + /******************************************************************************* + ** Setter for password + *******************************************************************************/ + public void setPassword(String password) + { + this.password = password; + } + + + + /******************************************************************************* + ** Fluent setter for password + *******************************************************************************/ + public SFTPBackendMetaData withPassword(String password) + { + this.password = password; + return (this); + } + + + + /******************************************************************************* + ** Getter for hostName + *******************************************************************************/ + public String getHostName() + { + return (this.hostName); + } + + + + /******************************************************************************* + ** Setter for hostName + *******************************************************************************/ + public void setHostName(String hostName) + { + this.hostName = hostName; + } + + + + /******************************************************************************* + ** Fluent setter for hostName + *******************************************************************************/ + public SFTPBackendMetaData withHostName(String hostName) + { + this.hostName = hostName; + return (this); + } + + + + /******************************************************************************* + ** Getter for port + *******************************************************************************/ + public Integer getPort() + { + return (this.port); + } + + + + /******************************************************************************* + ** Setter for port + *******************************************************************************/ + public void setPort(Integer port) + { + this.port = port; + } + + + + /******************************************************************************* + ** Fluent setter for port + *******************************************************************************/ + public SFTPBackendMetaData withPort(Integer port) + { + this.port = port; + return (this); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java new file mode 100644 index 00000000..b5205cb9 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java @@ -0,0 +1,38 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum SFTPBackendVariantSetting implements BackendVariantSetting +{ + USERNAME, + PASSWORD, + HOSTNAME, + PORT, + BASE_PATH +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java new file mode 100644 index 00000000..3a1605e0 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule; + + +/******************************************************************************* + ** SFTP specific Extension of QTableBackendDetails + *******************************************************************************/ +public class SFTPTableBackendDetails extends AbstractFilesystemTableBackendDetails +{ + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public SFTPTableBackendDetails() + { + super(); + setBackendType(SFTPBackendModule.class); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java new file mode 100644 index 00000000..19887c2a --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java @@ -0,0 +1,179 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.utils; + + +import java.io.IOException; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.apache.sshd.sftp.client.SftpClient; +import org.jetbrains.annotations.NotNull; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPOutputStream extends PipedOutputStream +{ + private static final QLogger LOG = QLogger.getLogger(SFTPOutputStream.class); + + private final SftpClient sftpClient; + + private final PipedInputStream pipedInputStream; + private final Future putFuture; + + private AtomicBoolean started = new AtomicBoolean(false); + private AtomicReference putException = new AtomicReference<>(null); + + + + /*************************************************************************** + ** + ***************************************************************************/ + public SFTPOutputStream(SftpClient sftpClient, String path) throws IOException + { + pipedInputStream = new PipedInputStream(this, 32 * 1024); + + this.sftpClient = sftpClient; + + putFuture = Executors.newSingleThreadExecutor().submit(() -> + { + try + { + started.set(true); + sftpClient.put(pipedInputStream, path); + } + catch(Exception e) + { + putException.set(e); + LOG.error("Error putting SFTP output stream", e); + + try + { + pipedInputStream.close(); + } + catch(IOException ex) + { + LOG.error("Secondary error closing pipedInputStream after sftp put error", e); + } + + throw new RuntimeException(e); + } + }); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void write(@NotNull byte[] b) throws IOException + { + try + { + super.write(b); + } + catch(IOException e) + { + if(putException.get() != null) + { + throw new IOException("Error performing SFTP put", putException.get()); + } + + throw new IOException("Error writing to SFTP output stream", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws IOException + { + try + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // don't try to close anything until we know that the sftpClient.put call's thread // + // has tried to start (otherwise, race condition could cause us to close things too early) // + ///////////////////////////////////////////////////////////////////////////////////////////// + int sleepLoops = 0; + while(!started.get() && sleepLoops++ <= 30) + { + SleepUtils.sleep(1, TimeUnit.SECONDS); + } + + /////////////////////////////////////////////////////////////////////////////////////// + // closing the pipedOutputStream (super) causes things to flush and complete the put // + /////////////////////////////////////////////////////////////////////////////////////// + super.close(); + + //////////////////////////////// + // wait for the put to finish // + //////////////////////////////// + putFuture.get(60 - sleepLoops, TimeUnit.SECONDS); + + /////////////////////////////////////////////////////////////////////////////// + // in case the put-future never did start, throw explicitly mentioning that. // + /////////////////////////////////////////////////////////////////////////////// + if(sleepLoops >= 30) + { + throw (new Exception("future to can sftpClient.put() was never started.")); + } + } + catch(ExecutionException ee) + { + throw new IOException("Error performing SFTP put", ee); + } + catch(Exception e) + { + if(putException.get() != null) + { + throw new IOException("Error performing SFTP put", putException.get()); + } + + throw new IOException("Error closing SFTP output stream", e); + } + finally + { + try + { + sftpClient.close(); + } + catch(IOException e) + { + LOG.error("Error closing SFTP client", e); + } + } + } + +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index acad9100..233cad0d 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.filesystem; import java.io.File; import java.io.IOException; import java.util.List; +import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -37,12 +38,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.FilesystemTableMetaDataBuilder; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; @@ -52,6 +55,10 @@ import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fil import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails; import org.apache.commons.io.FileUtils; @@ -60,20 +67,26 @@ import org.apache.commons.io.FileUtils; *******************************************************************************/ public class TestUtils { - public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; - public static final String BACKEND_NAME_S3 = "s3"; - public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; - public static final String BACKEND_NAME_MOCK = "mock"; - public static final String BACKEND_NAME_MEMORY = "memory"; + public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; + public static final String BACKEND_NAME_S3 = "s3"; + public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; + public static final String BACKEND_NAME_SFTP = "sftp"; + public static final String BACKEND_NAME_SFTP_WITH_VARIANTS = "sftpWithVariants"; + public static final String BACKEND_NAME_MOCK = "mock"; + public static final String BACKEND_NAME_MEMORY = "memory"; public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json"; public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv"; public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob"; public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive"; public static final String TABLE_NAME_PERSON_S3 = "person-s3"; + public static final String TABLE_NAME_PERSON_SFTP = "person-sftp"; public static final String TABLE_NAME_BLOB_S3 = "s3-blob"; public static final String TABLE_NAME_PERSON_MOCK = "person-mock"; public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix"; + public static final String TABLE_NAME_SFTP_FILE = "sftp-file"; + public static final String TABLE_NAME_SFTP_FILE_VARIANTS = "sftp-file-with-variants"; + public static final String TABLE_NAME_VARIANT_OPTIONS = "variant-options-table"; public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed"; public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter"; @@ -148,6 +161,7 @@ public class TestUtils qInstance.addBackend(defineS3Backend()); qInstance.addBackend(defineS3BackendSansPrefix()); qInstance.addTable(defineS3CSVPersonTable()); + qInstance.addTable(defineSFTPCSVPersonTable()); qInstance.addTable(defineS3BlobTable()); qInstance.addTable(defineS3BlobSansPrefixTable()); qInstance.addBackend(defineMockBackend()); @@ -155,6 +169,15 @@ public class TestUtils qInstance.addTable(defineMockPersonTable()); qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess()); + QBackendMetaData sftpBackend = defineSFTPBackend(); + qInstance.addBackend(sftpBackend); + qInstance.addTable(defineSFTPFileTable(sftpBackend)); + + QBackendMetaData sftpBackendWithVariants = defineSFTPBackendWithVariants(); + qInstance.addBackend(sftpBackendWithVariants); + qInstance.addTable(defineSFTPFileTableWithVariants(sftpBackendWithVariants)); + qInstance.addTable(defineVariantOptionsTable()); + definePersonCsvImporter(qInstance); return (qInstance); @@ -162,6 +185,21 @@ public class TestUtils + /*************************************************************************** + ** + ***************************************************************************/ + private static QTableMetaData defineVariantOptionsTable() + { + return new QTableMetaData() + .withName(TABLE_NAME_VARIANT_OPTIONS) + .withBackendName(defineMemoryBackend().getName()) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("basePath", QFieldType.STRING)); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -379,6 +417,25 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineSFTPCSVPersonTable() + { + return new QTableMetaData() + .withName(TABLE_NAME_PERSON_SFTP) + .withLabel("Person SFTP Table") + .withBackendName(BACKEND_NAME_SFTP) + .withPrimaryKeyField("id") + .withFields(defineCommonPersonTableFields()) + .withBackendDetails(new SFTPTableBackendDetails() + .withRecordFormat(RecordFormat.CSV) + .withCardinality(Cardinality.MANY) + ); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -463,4 +520,77 @@ public class TestUtils MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule(); return (mockAuthenticationModule.createSession(defineInstance(), null)); } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QTableMetaData defineSFTPFileTable(QBackendMetaData sftpBackend) + { + return new FilesystemTableMetaDataBuilder() + .withBasePath(BaseSFTPTest.TABLE_FOLDER) + .withBackend(sftpBackend) + .withName(TABLE_NAME_SFTP_FILE) + .buildStandardCardinalityOneTable() + .withLabel("SFTP Files"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QBackendMetaData defineSFTPBackend() + { + return (new SFTPBackendMetaData() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withHostName(BaseSFTPTest.HOST_NAME) + .withPort(BaseSFTPTest.getCurrentPort()) + .withBasePath(BaseSFTPTest.BACKEND_FOLDER) + .withName(BACKEND_NAME_SFTP)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QTableMetaData defineSFTPFileTableWithVariants(QBackendMetaData sftpBackend) + { + return new FilesystemTableMetaDataBuilder() + .withBasePath(BaseSFTPTest.TABLE_FOLDER) + .withBackend(sftpBackend) + .withName(TABLE_NAME_SFTP_FILE_VARIANTS) + .buildStandardCardinalityOneTable() + .withLabel("SFTP Files"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QBackendMetaData defineSFTPBackendWithVariants() + { + return (new SFTPBackendMetaData() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withHostName(BaseSFTPTest.HOST_NAME) + .withPort(BaseSFTPTest.getCurrentPort()) + + //////////////////////////////////// + // only get basePath from variant // + //////////////////////////////////// + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withOptionsTableName(TABLE_NAME_VARIANT_OPTIONS) + .withVariantTypeKey(TABLE_NAME_VARIANT_OPTIONS) + .withBackendSettingSourceFieldNameMap(Map.of( + SFTPBackendVariantSetting.BASE_PATH, "basePath" + )) + ) + .withName(BACKEND_NAME_SFTP_WITH_VARIANTS)); + } } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java new file mode 100644 index 00000000..69aa1e9e --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java @@ -0,0 +1,139 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp; + + +import com.kingsrook.qqq.backend.module.filesystem.BaseTest; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.MountableFile; + + +/******************************************************************************* + ** Base class for tests that want to be able to work with sftp testcontainer + *******************************************************************************/ +public class BaseSFTPTest extends BaseTest +{ + public static final int PORT = 22; + public static final String USERNAME = "testuser"; + public static final String PASSWORD = "testpass"; + public static final String HOST_NAME = "localhost"; + + public static final String BACKEND_FOLDER = "upload"; + public static final String TABLE_FOLDER = "files"; + public static final String REMOTE_DIR = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER; + + private static GenericContainer sftpContainer; + private static Integer currentPort; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @BeforeAll + static void setUp() throws Exception + { + sftpContainer = new GenericContainer<>("atmoz/sftp:latest") + .withExposedPorts(PORT) + .withCommand(USERNAME + ":" + PASSWORD + ":1001"); + + sftpContainer.start(); + + for(int i = 0; i < 5; i++) + { + sftpContainer.copyFileToContainer(MountableFile.forClasspathResource("files/testfile.txt"), REMOTE_DIR + "/testfile-" + i + ".txt"); + } + + grantUploadFilesDirWritePermission(); + + currentPort = sftpContainer.getMappedPort(22); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @AfterAll + static void tearDown() + { + if(sftpContainer != null) + { + sftpContainer.stop(); + } + } + + + + /******************************************************************************* + ** Getter for currentPort + ** + *******************************************************************************/ + public static Integer getCurrentPort() + { + return currentPort; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static void revokeUploadFilesDirWritePermission() throws Exception + { + setUploadFilesDirPermission("444"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static void grantUploadFilesDirWritePermission() throws Exception + { + setUploadFilesDirPermission("777"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void setUploadFilesDirPermission(String mode) throws Exception + { + sftpContainer.execInContainer("chmod", mode, "/home/testuser/upload/files"); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected void mkdirInSftpContainerUnderHomeTestuser(String path) throws Exception + { + Container.ExecResult mkdir = sftpContainer.execInContainer("mkdir", "-p", "/home/testuser/" + path); + System.out.println(mkdir.getExitCode()); + } +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java new file mode 100644 index 00000000..643fe2ba --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java @@ -0,0 +1,64 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPCountActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCount1() throws QException + { + CountInput countInput = initCountRequest(); + SFTPCountAction countAction = new SFTPCountAction(); + CountOutput countOutput = countAction.execute(countInput); + Assertions.assertEquals(5, countOutput.getCount(), "Expected # of rows from unfiltered count"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private CountInput initCountRequest() throws QException + { + CountInput countInput = new CountInput(); + countInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE); + return countInput; + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java new file mode 100644 index 00000000..3a992fbf --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPDeleteActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws QException + { + assertThrows(NotImplementedException.class, () -> new SFTPDeleteAction().execute(new DeleteInput())); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java new file mode 100644 index 00000000..c648021c --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java @@ -0,0 +1,120 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.IOException; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPInsertActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCardinalityOne() throws QException, IOException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE); + insertInput.setRecords(List.of( + new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.") + )); + + SFTPInsertAction insertAction = new SFTPInsertAction(); + + InsertOutput insertOutput = insertAction.execute(insertInput); + assertThat(insertOutput.getRecords()) + .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseSFTPTest.BACKEND_FOLDER)); + + QRecord record = insertOutput.getRecords().get(0); + String fullPath = record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH); + assertThat(record.getErrors()).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCardinalityOnePermissionError() throws Exception + { + try + { + revokeUploadFilesDirWritePermission(); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE); + insertInput.setRecords(List.of( + new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.") + )); + + SFTPInsertAction insertAction = new SFTPInsertAction(); + + InsertOutput insertOutput = insertAction.execute(insertInput); + + QRecord record = insertOutput.getRecords().get(0); + assertThat(record.getErrors()).isNotEmpty(); + assertThat(record.getErrors().get(0).getMessage()).contains("Error writing file: Permission denied"); + } + finally + { + grantUploadFilesDirWritePermission(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCardinalityMany() throws QException, IOException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_SFTP); + insertInput.setRecords(List.of( + new QRecord().withValue("id", "1").withValue("firstName", "Bob") + )); + + SFTPInsertAction insertAction = new SFTPInsertAction(); + + assertThatThrownBy(() -> insertAction.execute(insertInput)) + .hasRootCauseInstanceOf(NotImplementedException.class); + } +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java new file mode 100644 index 00000000..fdce8969 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java @@ -0,0 +1,105 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** Unit test for SFTPQueryAction + *******************************************************************************/ +class SFTPQueryActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQuery1() throws QException + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQueryVariantsTable() throws Exception + { + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_VARIANT_OPTIONS).withRecords(List.of( + new QRecord().withValue("id", 1).withValue("basePath", BaseSFTPTest.BACKEND_FOLDER), + new QRecord().withValue("id", 2).withValue("basePath", "empty-folder"), + new QRecord().withValue("id", 3).withValue("basePath", "non-existing-path") + ))); + + mkdirInSftpContainerUnderHomeTestuser("empty-folder/files"); + + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS); + assertThatThrownBy(() -> new QueryAction().execute(queryInput)) + .hasMessageContaining("Could not find Backend Variant information for Backend"); + + QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 1)); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + + QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 2)); + queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(0, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + + QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 3)); + assertThatThrownBy(() -> new QueryAction().execute(queryInput)) + .rootCause() + .hasMessageContaining("No such file"); + + // Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QueryInput initQueryRequest() throws QException + { + QueryInput queryInput = new QueryInput(); + return queryInput; + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java new file mode 100644 index 00000000..ed49cc2d --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java @@ -0,0 +1,115 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for FilesystemStorageAction + *******************************************************************************/ +public class SFTPStorageActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testSmall() throws Exception + { + String data = "Hellooo, Storage."; + runTest(data); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testPermissionError() throws Exception + { + try + { + revokeUploadFilesDirWritePermission(); + String data = "oops!"; + assertThatThrownBy(() -> runTest(data)) + .hasRootCauseInstanceOf(IOException.class) + .rootCause() + .hasMessageContaining("Permission denied"); + } + finally + { + grantUploadFilesDirWritePermission(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testLarge() throws Exception + { + String data = StringUtils.join("!", Collections.nCopies(5_000_000, "Hello")); + runTest(data); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void runTest(String data) throws QException, IOException + { + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_SFTP_FILE).withReference("fromStorageAction.txt"); + + StorageAction storageAction = new StorageAction(); + OutputStream outputStream = storageAction.createOutputStream(storageInput); + outputStream.write(data.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + InputStream inputStream = storageAction.getInputStream(storageInput); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + inputStream.transferTo(byteArrayOutputStream); + + assertEquals(data.length(), byteArrayOutputStream.toString(StandardCharsets.UTF_8).length()); + assertEquals(data, byteArrayOutputStream.toString(StandardCharsets.UTF_8)); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java new file mode 100644 index 00000000..2f025320 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPUpdateActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws QException + { + assertThrows(NotImplementedException.class, () -> new SFTPUpdateAction().execute(new UpdateInput())); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/resources/files/testfile.txt b/qqq-backend-module-filesystem/src/test/resources/files/testfile.txt new file mode 100644 index 00000000..eab448b1 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/resources/files/testfile.txt @@ -0,0 +1,3 @@ +This is a file. + +It is a test.