QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-19 05:30:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Initial add of sftp filesystem module

Browse Source
This commit is contained in:
darin.kelkhoff
2025-02-14 20:26:44 -06:00
parent 5a7199495d
commit b93114a9ba
23 changed files with 2176 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.filesystem;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@ -37,12 +38,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.FilesystemTableMetaDataBuilder;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
@ -52,6 +55,10 @@ import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fil
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting;
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails;
import org.apache.commons.io.FileUtils;
@ -60,20 +67,26 @@ import org.apache.commons.io.FileUtils;
*******************************************************************************/
public class TestUtils
{
public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
public static final String BACKEND_NAME_S3 = "s3";
public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix";
public static final String BACKEND_NAME_MOCK = "mock";
public static final String BACKEND_NAME_MEMORY = "memory";
public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
public static final String BACKEND_NAME_S3 = "s3";
public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix";
public static final String BACKEND_NAME_SFTP = "sftp";
public static final String BACKEND_NAME_SFTP_WITH_VARIANTS = "sftpWithVariants";
public static final String BACKEND_NAME_MOCK = "mock";
public static final String BACKEND_NAME_MEMORY = "memory";
public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json";
public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv";
public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob";
public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive";
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
public static final String TABLE_NAME_PERSON_SFTP = "person-sftp";
public static final String TABLE_NAME_BLOB_S3 = "s3-blob";
public static final String TABLE_NAME_PERSON_MOCK = "person-mock";
public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix";
public static final String TABLE_NAME_SFTP_FILE = "sftp-file";
public static final String TABLE_NAME_SFTP_FILE_VARIANTS = "sftp-file-with-variants";
public static final String TABLE_NAME_VARIANT_OPTIONS = "variant-options-table";
public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed";
public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter";
@ -148,6 +161,7 @@ public class TestUtils
qInstance.addBackend(defineS3Backend());
qInstance.addBackend(defineS3BackendSansPrefix());
qInstance.addTable(defineS3CSVPersonTable());
qInstance.addTable(defineSFTPCSVPersonTable());
qInstance.addTable(defineS3BlobTable());
qInstance.addTable(defineS3BlobSansPrefixTable());
qInstance.addBackend(defineMockBackend());
@ -155,6 +169,15 @@ public class TestUtils
qInstance.addTable(defineMockPersonTable());
qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
QBackendMetaData sftpBackend = defineSFTPBackend();
qInstance.addBackend(sftpBackend);
qInstance.addTable(defineSFTPFileTable(sftpBackend));
QBackendMetaData sftpBackendWithVariants = defineSFTPBackendWithVariants();
qInstance.addBackend(sftpBackendWithVariants);
qInstance.addTable(defineSFTPFileTableWithVariants(sftpBackendWithVariants));
qInstance.addTable(defineVariantOptionsTable());
definePersonCsvImporter(qInstance);
return (qInstance);
@ -162,6 +185,21 @@ public class TestUtils
/***************************************************************************
**
***************************************************************************/
private static QTableMetaData defineVariantOptionsTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_VARIANT_OPTIONS)
.withBackendName(defineMemoryBackend().getName())
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("basePath", QFieldType.STRING));
}
/*******************************************************************************
**
*******************************************************************************/
@ -379,6 +417,25 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineSFTPCSVPersonTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_PERSON_SFTP)
.withLabel("Person SFTP Table")
.withBackendName(BACKEND_NAME_SFTP)
.withPrimaryKeyField("id")
.withFields(defineCommonPersonTableFields())
.withBackendDetails(new SFTPTableBackendDetails()
.withRecordFormat(RecordFormat.CSV)
.withCardinality(Cardinality.MANY)
);
}
/*******************************************************************************
**
*******************************************************************************/
@ -463,4 +520,77 @@ public class TestUtils
MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule();
return (mockAuthenticationModule.createSession(defineInstance(), null));
}
/***************************************************************************
**
***************************************************************************/
public static QTableMetaData defineSFTPFileTable(QBackendMetaData sftpBackend)
{
return new FilesystemTableMetaDataBuilder()
.withBasePath(BaseSFTPTest.TABLE_FOLDER)
.withBackend(sftpBackend)
.withName(TABLE_NAME_SFTP_FILE)
.buildStandardCardinalityOneTable()
.withLabel("SFTP Files");
}
/***************************************************************************
**
***************************************************************************/
private static QBackendMetaData defineSFTPBackend()
{
return (new SFTPBackendMetaData()
.withUsername(BaseSFTPTest.USERNAME)
.withPassword(BaseSFTPTest.PASSWORD)
.withHostName(BaseSFTPTest.HOST_NAME)
.withPort(BaseSFTPTest.getCurrentPort())
.withBasePath(BaseSFTPTest.BACKEND_FOLDER)
.withName(BACKEND_NAME_SFTP));
}
/***************************************************************************
**
***************************************************************************/
public static QTableMetaData defineSFTPFileTableWithVariants(QBackendMetaData sftpBackend)
{
return new FilesystemTableMetaDataBuilder()
.withBasePath(BaseSFTPTest.TABLE_FOLDER)
.withBackend(sftpBackend)
.withName(TABLE_NAME_SFTP_FILE_VARIANTS)
.buildStandardCardinalityOneTable()
.withLabel("SFTP Files");
}
/***************************************************************************
**
***************************************************************************/
private static QBackendMetaData defineSFTPBackendWithVariants()
{
return (new SFTPBackendMetaData()
.withUsername(BaseSFTPTest.USERNAME)
.withPassword(BaseSFTPTest.PASSWORD)
.withHostName(BaseSFTPTest.HOST_NAME)
.withPort(BaseSFTPTest.getCurrentPort())
////////////////////////////////////
// only get basePath from variant //
////////////////////////////////////
.withUsesVariants(true)
.withBackendVariantsConfig(new BackendVariantsConfig()
.withOptionsTableName(TABLE_NAME_VARIANT_OPTIONS)
.withVariantTypeKey(TABLE_NAME_VARIANT_OPTIONS)
.withBackendSettingSourceFieldNameMap(Map.of(
SFTPBackendVariantSetting.BASE_PATH, "basePath"
))
)
.withName(BACKEND_NAME_SFTP_WITH_VARIANTS));
}
}

139
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java Normal file
View File

@ -0,0 +1,139 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp;
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.containers.Container;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.utility.MountableFile;
/*******************************************************************************
** Base class for tests that want to be able to work with sftp testcontainer
*******************************************************************************/
public class BaseSFTPTest extends BaseTest
{
public static final int PORT = 22;
public static final String USERNAME = "testuser";
public static final String PASSWORD = "testpass";
public static final String HOST_NAME = "localhost";
public static final String BACKEND_FOLDER = "upload";
public static final String TABLE_FOLDER = "files";
public static final String REMOTE_DIR = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER;
private static GenericContainer<?> sftpContainer;
private static Integer currentPort;
/***************************************************************************
**
***************************************************************************/
@BeforeAll
static void setUp() throws Exception
{
sftpContainer = new GenericContainer<>("atmoz/sftp:latest")
.withExposedPorts(PORT)
.withCommand(USERNAME + ":" + PASSWORD + ":1001");
sftpContainer.start();
for(int i = 0; i < 5; i++)
{
sftpContainer.copyFileToContainer(MountableFile.forClasspathResource("files/testfile.txt"), REMOTE_DIR + "/testfile-" + i + ".txt");
}
grantUploadFilesDirWritePermission();
currentPort = sftpContainer.getMappedPort(22);
}
/***************************************************************************
**
***************************************************************************/
@AfterAll
static void tearDown()
{
if(sftpContainer != null)
{
sftpContainer.stop();
}
}
/*******************************************************************************
** Getter for currentPort
**
*******************************************************************************/
public static Integer getCurrentPort()
{
return currentPort;
}
/***************************************************************************
**
***************************************************************************/
protected static void revokeUploadFilesDirWritePermission() throws Exception
{
setUploadFilesDirPermission("444");
}
/***************************************************************************
**
***************************************************************************/
protected static void grantUploadFilesDirWritePermission() throws Exception
{
setUploadFilesDirPermission("777");
}
/***************************************************************************
**
***************************************************************************/
private static void setUploadFilesDirPermission(String mode) throws Exception
{
sftpContainer.execInContainer("chmod", mode, "/home/testuser/upload/files");
}
/***************************************************************************
*
***************************************************************************/
protected void mkdirInSftpContainerUnderHomeTestuser(String path) throws Exception
{
Container.ExecResult mkdir = sftpContainer.execInContainer("mkdir", "-p", "/home/testuser/" + path);
System.out.println(mkdir.getExitCode());
}
}

64
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java Normal file
View File

@ -0,0 +1,64 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPCountActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCount1() throws QException
{
CountInput countInput = initCountRequest();
SFTPCountAction countAction = new SFTPCountAction();
CountOutput countOutput = countAction.execute(countInput);
Assertions.assertEquals(5, countOutput.getCount(), "Expected # of rows from unfiltered count");
}
/*******************************************************************************
**
*******************************************************************************/
private CountInput initCountRequest() throws QException
{
CountInput countInput = new CountInput();
countInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE);
return countInput;
}
}

48
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java Normal file
View File

@ -0,0 +1,48 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertThrows;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPDeleteActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void test() throws QException
{
assertThrows(NotImplementedException.class, () -> new SFTPDeleteAction().execute(new DeleteInput()));
}
}

120
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java Normal file
View File

@ -0,0 +1,120 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import java.io.IOException;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPInsertActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCardinalityOne() throws QException, IOException
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
));
SFTPInsertAction insertAction = new SFTPInsertAction();
InsertOutput insertOutput = insertAction.execute(insertInput);
assertThat(insertOutput.getRecords())
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseSFTPTest.BACKEND_FOLDER));
QRecord record = insertOutput.getRecords().get(0);
String fullPath = record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
assertThat(record.getErrors()).isNullOrEmpty();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCardinalityOnePermissionError() throws Exception
{
try
{
revokeUploadFilesDirWritePermission();
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
));
SFTPInsertAction insertAction = new SFTPInsertAction();
InsertOutput insertOutput = insertAction.execute(insertInput);
QRecord record = insertOutput.getRecords().get(0);
assertThat(record.getErrors()).isNotEmpty();
assertThat(record.getErrors().get(0).getMessage()).contains("Error writing file: Permission denied");
}
finally
{
grantUploadFilesDirWritePermission();
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCardinalityMany() throws QException, IOException
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_SFTP);
insertInput.setRecords(List.of(
new QRecord().withValue("id", "1").withValue("firstName", "Bob")
));
SFTPInsertAction insertAction = new SFTPInsertAction();
assertThatThrownBy(() -> insertAction.execute(insertInput))
.hasRootCauseInstanceOf(NotImplementedException.class);
}
}

105
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java Normal file
View File

@ -0,0 +1,105 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Unit test for SFTPQueryAction
*******************************************************************************/
class SFTPQueryActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQuery1() throws QException
{
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQueryVariantsTable() throws Exception
{
new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_VARIANT_OPTIONS).withRecords(List.of(
new QRecord().withValue("id", 1).withValue("basePath", BaseSFTPTest.BACKEND_FOLDER),
new QRecord().withValue("id", 2).withValue("basePath", "empty-folder"),
new QRecord().withValue("id", 3).withValue("basePath", "non-existing-path")
)));
mkdirInSftpContainerUnderHomeTestuser("empty-folder/files");
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS);
assertThatThrownBy(() -> new QueryAction().execute(queryInput))
.hasMessageContaining("Could not find Backend Variant information for Backend");
QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 1));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 2));
queryOutput = new QueryAction().execute(queryInput);
Assertions.assertEquals(0, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 3));
assertThatThrownBy(() -> new QueryAction().execute(queryInput))
.rootCause()
.hasMessageContaining("No such file");
// Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
}
/*******************************************************************************
**
*******************************************************************************/
private QueryInput initQueryRequest() throws QException
{
QueryInput queryInput = new QueryInput();
return queryInput;
}
}

115
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java Normal file
View File

@ -0,0 +1,115 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for FilesystemStorageAction
*******************************************************************************/
public class SFTPStorageActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testSmall() throws Exception
{
String data = "Hellooo, Storage.";
runTest(data);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testPermissionError() throws Exception
{
try
{
revokeUploadFilesDirWritePermission();
String data = "oops!";
assertThatThrownBy(() -> runTest(data))
.hasRootCauseInstanceOf(IOException.class)
.rootCause()
.hasMessageContaining("Permission denied");
}
finally
{
grantUploadFilesDirWritePermission();
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testLarge() throws Exception
{
String data = StringUtils.join("!", Collections.nCopies(5_000_000, "Hello"));
runTest(data);
}
/***************************************************************************
**
***************************************************************************/
private static void runTest(String data) throws QException, IOException
{
StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_SFTP_FILE).withReference("fromStorageAction.txt");
StorageAction storageAction = new StorageAction();
OutputStream outputStream = storageAction.createOutputStream(storageInput);
outputStream.write(data.getBytes(StandardCharsets.UTF_8));
outputStream.close();
InputStream inputStream = storageAction.getInputStream(storageInput);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
inputStream.transferTo(byteArrayOutputStream);
assertEquals(data.length(), byteArrayOutputStream.toString(StandardCharsets.UTF_8).length());
assertEquals(data, byteArrayOutputStream.toString(StandardCharsets.UTF_8));
}
}

48
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java Normal file
View File

@ -0,0 +1,48 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertThrows;
/*******************************************************************************
**
*******************************************************************************/
public class SFTPUpdateActionTest extends BaseSFTPTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void test() throws QException
{
assertThrows(NotImplementedException.class, () -> new SFTPUpdateAction().execute(new UpdateInput()));
}
}

3
qqq-backend-module-filesystem/src/test/resources/files/testfile.txt Normal file
View File

@ -0,0 +1,3 @@
This is a file.
It is a test.