QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-19 13:40:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Moving qqq-backend-module-filesystem into its own subdir

Browse Source
This commit is contained in:
darin.kelkhoff
2022-07-28 12:02:57 -05:00
parent ce36950247
commit 600048cc76
53 changed files with 0 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

228
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java Normal file
View File

@ -0,0 +1,228 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem;
import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationModule;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import org.apache.commons.io.FileUtils;
/*******************************************************************************
** Utility methods for Filesystem module tests
*******************************************************************************/
public class TestUtils
{
public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
public static final String BACKEND_NAME_S3 = "s3";
public static final String TABLE_NAME_PERSON_LOCAL_FS = "person";
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
///////////////////////////////////////////////////////////////////
// shouldn't be accessed directly, as we append a counter to it. //
///////////////////////////////////////////////////////////////////
public static final String BASE_PATH = "/tmp/filesystem-tests";
//////////////////////////////////////////////////////////////////////////////
// Used to make each test method have a unique folder path, more or less... //
// See methods that work with it. //
//////////////////////////////////////////////////////////////////////////////
private static int testInstanceCounter = 0;
/*******************************************************************************
** Meant to be called in a @BeforeEach - increment an internal counter that will
** give us a unique directory name for each test method.
*******************************************************************************/
public static void increaseTestInstanceCounter()
{
testInstanceCounter++;
}
/*******************************************************************************
** Reset the counter to 0 (e.g., to let some tests have a known value).
*******************************************************************************/
public static void resetTestInstanceCounter()
{
testInstanceCounter = 0;
}
/*******************************************************************************
** Get the current value of the testInstanceCounter. See {@link #increaseTestInstanceCounter()}
*******************************************************************************/
public static int getTestInstanceCounter()
{
return (testInstanceCounter);
}
/*******************************************************************************
** Meant to run both after and before test methods - makes sure the file system
** is empty for the path under the instance.
*******************************************************************************/
public static void cleanInstanceFiles() throws IOException
{
FileUtils.deleteDirectory(new File(BASE_PATH + File.separator + testInstanceCounter));
}
/*******************************************************************************
**
*******************************************************************************/
public static QInstance defineInstance() throws QInstanceValidationException
{
QInstance qInstance = new QInstance();
qInstance.setAuthentication(defineAuthentication());
qInstance.addBackend(defineLocalFilesystemBackend());
qInstance.addTable(defineLocalFilesystemJSONPersonTable());
qInstance.addBackend(defineS3Backend());
qInstance.addTable(defineS3CSVPersonTable());
new QInstanceValidator().validate(qInstance);
return (qInstance);
}
/*******************************************************************************
** Define the authentication used in standard tests - using 'mock' type.
**
*******************************************************************************/
public static QAuthenticationMetaData defineAuthentication()
{
return new QAuthenticationMetaData()
.withName("mock")
.withType(QAuthenticationType.MOCK);
}
/*******************************************************************************
**
*******************************************************************************/
public static FilesystemBackendMetaData defineLocalFilesystemBackend()
{
return (new FilesystemBackendMetaData()
.withBasePath(BASE_PATH + File.separator + testInstanceCounter)
.withName(BACKEND_NAME_LOCAL_FS));
}
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineLocalFilesystemJSONPersonTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_PERSON_LOCAL_FS)
.withLabel("Person")
.withBackendName(defineLocalFilesystemBackend().getName())
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"))
.withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"))
.withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"))
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
.withField(new QFieldMetaData("email", QFieldType.STRING))
.withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath("persons")
.withRecordFormat(RecordFormat.JSON)
.withCardinality(Cardinality.MANY)
);
}
/*******************************************************************************
**
*******************************************************************************/
public static S3BackendMetaData defineS3Backend()
{
return (new S3BackendMetaData()
.withBucketName(BaseS3Test.BUCKET_NAME)
.withBasePath(BaseS3Test.TEST_FOLDER)
.withName(BACKEND_NAME_S3));
}
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineS3CSVPersonTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_PERSON_S3)
.withLabel("Person S3 Table")
.withBackendName(defineS3Backend().getName())
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"))
.withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"))
.withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"))
.withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"))
.withField(new QFieldMetaData("email", QFieldType.STRING))
.withBackendDetails(new S3TableBackendDetails()
.withRecordFormat(RecordFormat.CSV)
.withCardinality(Cardinality.MANY)
);
}
/*******************************************************************************
**
*******************************************************************************/
public static QSession getMockSession() throws QInstanceValidationException
{
MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule();
return (mockAuthenticationModule.createSession(defineInstance(), null));
}
}

175
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java Normal file
View File

@ -0,0 +1,175 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local;
import java.io.File;
import java.io.IOException;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for FilesystemBackendModule
*******************************************************************************/
public class FilesystemBackendModuleTest
{
private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist";
@BeforeEach
public void beforeEach() throws IOException
{
new FilesystemActionTest().primeFilesystem();
}
@AfterEach
public void afterEach() throws Exception
{
new FilesystemActionTest().cleanFilesystem();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDeleteFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
/////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then delete one, then re-list, and assert that we have one fewer //
/////////////////////////////////////////////////////////////////////////////////////////////
List<File> filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
filesystemBackendModule.getActionBase().deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath());
List<File> filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeDelete.size() - 1, filesAfterDelete.size(),
"Should be one fewer file listed after deleting one.");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDeleteFileDoesNotExist() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
// note, our contract is that as long as the file doesn't exist after calling delete (e.g., if it wasn't there //
// to begin with, then we're okay, and don't expect an exception //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<File> filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
filesystemBackendModule.getActionBase().deleteFile(qInstance, table, PATH_THAT_WONT_EXIST);
List<File> filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeDelete.size(), filesAfterDelete.size(),
"Should be same number of files after deleting bogus path");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMoveFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
String subPath = basePath + File.separator + "subdir";
////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files (non-recursively) - then move one into a sub-folder, then re-list, and //
// assert that we have one fewer then list again including sub-folders, and see the changed count //
////////////////////////////////////////////////////////////////////////////////////////////////////
List<File> filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
String originalFilePath = filesBeforeMove.get(0).getAbsolutePath();
String movedFilePath = originalFilePath.replace(basePath, subPath);
filesystemBackendModule.getActionBase().moveFile(qInstance, table, originalFilePath, movedFilePath);
List<File> filesAfterMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeMove.size() - 1, filesAfterMove.size(), "Should be one fewer file in the listing after moving one.");
//////////////////////////////////////////////////////////////////////////
// move the file back and assert that the count goes back to the before //
//////////////////////////////////////////////////////////////////////////
filesystemBackendModule.getActionBase().moveFile(qInstance, table, movedFilePath, originalFilePath);
List<File> filesAfterMoveBack = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
Assertions.assertEquals(filesBeforeMove.size(), filesAfterMoveBack.size(), "Should be original number of files after moving back");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMoveFileDoesNotExit() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
String subPath = basePath + File.separator + "subdir";
List<File> filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
String originalFilePath = filesBeforeMove.get(0).getAbsolutePath();
String movedFilePath = originalFilePath.replace(basePath, subPath);
Assertions.assertThrows(FilesystemException.class, () ->
{
FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule();
filesystemBackendModule.getActionBase().moveFile(qInstance, table, PATH_THAT_WONT_EXIST, movedFilePath);
});
}
}

128
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java Normal file
View File

@ -0,0 +1,128 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import static org.junit.jupiter.api.Assertions.fail;
/*******************************************************************************
** Base class for Filesystem action tests.
**
** Knows how to set up the filesystem for the tests.
*******************************************************************************/
public class FilesystemActionTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
public void beforeEach() throws Exception
{
primeFilesystem();
}
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
public void afterEach() throws Exception
{
cleanFilesystem();
}
/*******************************************************************************
** Set up the file system
*******************************************************************************/
public void primeFilesystem() throws IOException
{
TestUtils.cleanInstanceFiles();
TestUtils.increaseTestInstanceCounter();
FilesystemBackendMetaData filesystemBackendMetaData = TestUtils.defineLocalFilesystemBackend();
File baseDirectory = new File(filesystemBackendMetaData.getBasePath());
boolean mkdirsResult = baseDirectory.mkdirs();
if(!mkdirsResult)
{
fail("Failed to make directories at [" + baseDirectory + "] for filesystem backend module");
}
writePersonFiles(baseDirectory);
}
/*******************************************************************************
** Write some data files into the directory for the filesystem module.
*******************************************************************************/
private void writePersonFiles(File baseDirectory) throws IOException
{
String fullPath = baseDirectory.getAbsolutePath();
if (TestUtils.defineLocalFilesystemJSONPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
{
if (StringUtils.hasContent(details.getBasePath()))
{
fullPath += File.separatorChar + details.getBasePath();
}
}
fullPath += File.separatorChar;
String jsonData1 = """
[
{"id":1,"createDate":"2021-10-26 14:39:37","modifyDate":"2021-10-26 14:39:37","firstName":"John","lastName":"Doe","birthDate":"1981-01-01","email":"john@kingsrook.com"},
{"id":2,"createDate":"2022-06-17 14:52:59","modifyDate":"2022-06-17 14:52:59","firstName":"Jane","lastName":"Smith","birthDate":"1982-02-02","email":"jane@kingsrook.com"}
]
""";
FileUtils.writeStringToFile(new File(fullPath + "DATA-1.json"), jsonData1);
String jsonData2 = """
[
{"id":3,"createDate":"2021-11-27 15:40:38","modifyDate":"2021-11-27 15:40:38","firstName":"Homer","lastName":"S","birthDate":"1983-03-03","email":"homer.s@kingsrook.com"}
]
""";
FileUtils.writeStringToFile(new File(fullPath + "DATA-2.json"), jsonData2);
}
/*******************************************************************************
**
*******************************************************************************/
public void cleanFilesystem() throws IOException
{
TestUtils.cleanInstanceFiles();
}
}

100
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java Normal file
View File

@ -0,0 +1,100 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/*******************************************************************************
**
*******************************************************************************/
public class FilesystemQueryActionTest extends FilesystemActionTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQuery1() throws QException
{
QueryInput queryInput = new QueryInput();
queryInput.setInstance(TestUtils.defineInstance());
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
Assertions.assertTrue(queryOutput.getRecords().stream()
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
"All records should have a full-path in their backend details, matching the test folder name");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQueryWithFileCustomizer() throws QException
{
QueryInput queryInput = new QueryInput();
QInstance instance = TestUtils.defineInstance();
QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
table.withCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ, new QCodeReference()
.withName(ValueUpshifter.class.getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.CUSTOMIZER));
queryInput.setInstance(instance);
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
Assertions.assertTrue(
queryOutput.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")),
"All records should have their email addresses up-shifted.");
}
public static class ValueUpshifter implements Function<String, String>
{
@Override
public String apply(String s)
{
return (s.replaceAll("kingsrook.com", "KINGSROOK.COM"));
}
}
}

76
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java Normal file
View File

@ -0,0 +1,76 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for FilesystemBackendMetaData
*******************************************************************************/
class FilesystemBackendMetaDataTest
{
/*******************************************************************************
** Test that an instance can be serialized as expected
*******************************************************************************/
@Test
public void testSerializingToJson() throws QInstanceValidationException
{
TestUtils.resetTestInstanceCounter();
QInstance qInstance = TestUtils.defineInstance();
String json = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance);
System.out.println(JsonUtils.prettyPrint(json));
System.out.println(json);
String expectToContain = """
"local-filesystem":{"basePath":"/tmp/filesystem-tests/0","backendType":"filesystem","name":"local-filesystem"}""";
assertTrue(json.contains(expectToContain));
}
/*******************************************************************************
** Test that an instance can be deserialized as expected
*******************************************************************************/
@Test
public void testDeserializingFromJson() throws IOException, QInstanceValidationException
{
QInstanceAdapter qInstanceAdapter = new QInstanceAdapter();
QInstance qInstance = TestUtils.defineInstance();
String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance);
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
assertThat(deserialized.getBackends()).usingRecursiveComparison()
.isEqualTo(qInstance.getBackends());
}
}

228
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java Normal file
View File

@ -0,0 +1,228 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
import java.io.File;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for BasicETLCleanupSourceFilesFunction
*******************************************************************************/
public class BasicETLCleanupSourceFilesStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDelete1Record1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
testDelete(qInstance, List.of(filePath));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDelete2Records1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
testDelete(qInstance, List.of(filePath, filePath));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDelete2Record2File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath1 = getRandomFilePathPersonTable(qInstance);
String filePath2 = getRandomFilePathPersonTable(qInstance);
testDelete(qInstance, List.of(filePath1, filePath2));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMove1Record1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
testMove(qInstance, List.of(filePath));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMove2Records1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
testMove(qInstance, List.of(filePath, filePath));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMove2Record2File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath1 = getRandomFilePathPersonTable(qInstance);
String filePath2 = getRandomFilePathPersonTable(qInstance);
testMove(qInstance, List.of(filePath1, filePath2));
}
/*******************************************************************************
**
*******************************************************************************/
private void testDelete(QInstance qInstance, List<String> filePaths) throws Exception
{
RunBackendStepOutput runBackendStepOutput = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesStep.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesStep.VALUE_DELETE,
// todo - even though this field isn't needed, since we gave a value of "delete"
// the RunFunctionAction considers any missing input to be an error...
BasicETLCleanupSourceFilesStep.FIELD_DESTINATION_FOR_MOVES, ""));
assertNull(runBackendStepOutput.getException());
for(String filePath : filePaths)
{
assertFalse(new File(filePath).exists(), "File should have been deleted.");
}
}
/*******************************************************************************
**
*******************************************************************************/
private void testMove(QInstance qInstance, List<String> filePaths) throws Exception
{
String trashDir = File.separator + "tmp" + File.separator + "trash";
RunBackendStepOutput runBackendStepOutput = runFunction(qInstance, filePaths, Map.of(
BasicETLCleanupSourceFilesStep.FIELD_MOVE_OR_DELETE, BasicETLCleanupSourceFilesStep.VALUE_MOVE,
BasicETLCleanupSourceFilesStep.FIELD_DESTINATION_FOR_MOVES, trashDir));
assertNull(runBackendStepOutput.getException());
for(String filePath : filePaths)
{
assertFalse(new File(filePath).exists(), "File should have been moved.");
String movedPath = trashDir + File.separator + (new File(filePath).getName());
assertTrue(new File(movedPath).exists(), "File should have been moved.");
}
}
/*******************************************************************************
**
*******************************************************************************/
private RunBackendStepOutput runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception
{
QBackendStepMetaData backendStepMetaData = new BasicETLCleanupSourceFilesStep().defineStepMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
qInstance.addProcess(qProcessMetaData);
HashSet<String> filePathsSet = new HashSet<>(filePaths);
for(String filePath : filePathsSet)
{
File file = new File(filePath);
FileUtils.writeStringToFile(file, "content");
}
// List<QRecord> records = filePaths.stream()
// .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList();
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(qInstance);
runBackendStepInput.setStepName(backendStepMetaData.getName());
runBackendStepInput.setProcessName(qProcessMetaData.getName());
// runFunctionRequest.setRecords(records);
runBackendStepInput.setSession(TestUtils.getMockSession());
runBackendStepInput.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
runBackendStepInput.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
runBackendStepInput.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));
for(Map.Entry<String, String> entry : values.entrySet())
{
runBackendStepInput.addValue(entry.getKey(), entry.getValue());
}
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
return (runFunctionAction.execute(runBackendStepInput));
}
/*******************************************************************************
**
*******************************************************************************/
private String getRandomFilePathPersonTable(QInstance qInstance)
{
FilesystemBackendMetaData backend = (FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS).getBackendDetails();
String tablePath = backend.getBasePath() + File.separator + backendDetails.getBasePath();
String filePath = tablePath + File.separator + UUID.randomUUID();
return filePath;
}
}

115
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStepTest.java Normal file
View File

@ -0,0 +1,115 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
import java.util.Arrays;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for BasicETLCollectSourceFileNamesFunction
*******************************************************************************/
class BasicETLCollectSourceFileNamesStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testOneFile() throws Exception
{
String file = "/tmp/test1.csv";
String result = runTest(file);
assertEquals(file, result);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testTwoFiles() throws Exception
{
String file1 = "/tmp/test1.csv";
String file2 = "/tmp/test2.csv";
String result = runTest(file1, file2);
//////////////////////////////////////////////////////////////////////
// the names go into a set, so they can come out in either order... //
//////////////////////////////////////////////////////////////////////
assertTrue(result.equals(file1 + "," + file2) || result.equals((file2 + "," + file1)));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDuplicatedFile() throws Exception
{
String file = "/tmp/test1.csv";
String result = runTest(file, file);
assertEquals(file, result);
}
/*******************************************************************************
**
*******************************************************************************/
private String runTest(String... fileNames) throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QBackendStepMetaData backendStepMetaData = new BasicETLCollectSourceFileNamesStep().defineStepMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
qInstance.addProcess(qProcessMetaData);
List<QRecord> records = Arrays.stream(fileNames).map(fileName ->
new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fileName)).toList();
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(qInstance);
runBackendStepInput.setSession(TestUtils.getMockSession());
runBackendStepInput.setStepName(backendStepMetaData.getName());
runBackendStepInput.setProcessName(qProcessMetaData.getName());
runBackendStepInput.setRecords(records);
RunBackendStepAction runBackendStepAction = new RunBackendStepAction();
RunBackendStepOutput result = runBackendStepAction.execute(runBackendStepInput);
return ((String) result.getValues().get(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS));
}
}

266
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java Normal file
View File

@ -0,0 +1,266 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync;
import java.util.List;
import java.util.stream.Collectors;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModuleSubclassForTest;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for FilesystemSyncProcess using S3 backend
*******************************************************************************/
class FilesystemSyncProcessS3Test extends BaseS3Test
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void test() throws Exception
{
QBackendModuleDispatcher.registerBackendModule(new S3BackendModuleSubclassForTest());
QInstance qInstance = TestUtils.defineInstance();
String sourceBucket = "source-bucket";
String archiveBucket = "archive-bucket";
String processingBucket = "processing-bucket";
getAmazonS3().createBucket(sourceBucket);
getAmazonS3().createBucket(archiveBucket);
getAmazonS3().createBucket(processingBucket);
S3BackendMetaData sourceBackend = defineBackend(qInstance, "source", sourceBucket);
S3BackendMetaData archiveBackend = defineBackend(qInstance, "archive", archiveBucket);
S3BackendMetaData processingBackend = defineBackend(qInstance, "processing", processingBucket);
QTableMetaData sourceTable = defineTable(qInstance, "source", sourceBackend, "source", "*/l3/*.csv");
QTableMetaData archiveTable = defineTable(qInstance, "archive", archiveBackend, "archive", "*/l3/*.csv");
QTableMetaData processingTable = defineTable(qInstance, "processing", processingBackend, "processing", "**/*.csv");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QBackendStepMetaData step = process.getBackendStep(FilesystemSyncStep.STEP_NAME);
qInstance.addProcess(process);
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
///////////////////////////
// write some test files //
///////////////////////////
writeTestFile(sourceBackend, sourceTable, "foo/l3/1.csv", "x");
writeTestFile(sourceBackend, sourceTable, "bar/l3/2.csv", "x");
writeTestFile(archiveBackend, archiveTable, "foo/l3/1.csv", "x");
printTableListing(sourceBackend, sourceTable);
printTableListing(archiveBackend, archiveTable);
printTableListing(processingBackend, processingTable);
//////////////////
// run the step //
//////////////////
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(qInstance);
runBackendStepInput.setStepName(step.getName());
runBackendStepInput.setProcessName(process.getName());
runBackendStepInput.setSession(TestUtils.getMockSession());
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepOutput runBackendStepOutput = runFunctionAction.execute(runBackendStepInput);
// System.out.println(runBackendStepResult);
printTableListing(sourceBackend, sourceTable);
printTableListing(archiveBackend, archiveTable);
printTableListing(processingBackend, processingTable);
assertTableListing(archiveBackend, archiveTable, "root/archive/foo/l3/1.csv", "root/archive/bar/l3/2.csv");
assertTableListing(processingBackend, processingTable, "root/processing/bar/l3/2.csv");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testArchiveAndProcessingInSameBucket() throws Exception
{
QBackendModuleDispatcher.registerBackendModule(new S3BackendModuleSubclassForTest());
QInstance qInstance = TestUtils.defineInstance();
String vendorBucket = "vendor-bucket";
String localBucket = "local-bucket";
getAmazonS3().createBucket(vendorBucket);
getAmazonS3().createBucket(localBucket);
S3BackendMetaData vendorBackend = defineBackend(qInstance, "source", vendorBucket);
S3BackendMetaData localBackend = defineBackend(qInstance, "archive", localBucket);
QTableMetaData sourceTable = defineTable(qInstance, "source", vendorBackend, "source", "*/l3/*.csv");
QTableMetaData archiveTable = defineTable(qInstance, "archive", localBackend, "archive", "*/l3/*.csv");
QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QBackendStepMetaData step = process.getBackendStep(FilesystemSyncStep.STEP_NAME);
qInstance.addProcess(process);
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
///////////////////////////
// write some test files //
///////////////////////////
writeTestFile(vendorBackend, sourceTable, "foo/l3/1.csv", "x");
writeTestFile(vendorBackend, sourceTable, "bar/l3/2.csv", "x");
writeTestFile(localBackend, archiveTable, "foo/l3/1.csv", "x");
printTableListing(vendorBackend, sourceTable);
printTableListing(localBackend, archiveTable);
printTableListing(localBackend, processingTable);
//////////////////
// run the step //
//////////////////
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(qInstance);
runBackendStepInput.setStepName(step.getName());
runBackendStepInput.setProcessName(process.getName());
runBackendStepInput.setSession(TestUtils.getMockSession());
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepOutput runBackendStepOutput = runFunctionAction.execute(runBackendStepInput);
// System.out.println(runBackendStepResult);
printTableListing(vendorBackend, sourceTable);
printTableListing(localBackend, archiveTable);
printTableListing(localBackend, processingTable);
assertTableListing(localBackend, archiveTable, "root/archive/foo/l3/1.csv", "root/archive/bar/l3/2.csv");
assertTableListing(localBackend, processingTable, "root/processing/bar/l3/2.csv");
}
/*******************************************************************************
**
*******************************************************************************/
private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException
{
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
List<S3ObjectSummary> s3ObjectSummaries = actionBase.listFiles(table, backend);
assertEquals(paths.length, s3ObjectSummaries.size(), "Expected number of files in table: " + table.getName());
for(String path : paths)
{
assertTrue(s3ObjectSummaries.stream().anyMatch(s3o -> s3o.getKey().equals(path)),
"Path [" + path + "] should be in the listing, but was not. Full listing is: " +
s3ObjectSummaries.stream().map(S3ObjectSummary::getKey).collect(Collectors.joining(",")));
}
}
/*******************************************************************************
**
*******************************************************************************/
private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException
{
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
System.out.println("Files in: " + table.getName());
actionBase.listFiles(table, backend).forEach(o -> System.out.println(o.getKey()));
}
/*******************************************************************************
**
*******************************************************************************/
private void writeTestFile(S3BackendMetaData backend, QTableMetaData table, String name, String content) throws Exception
{
S3BackendModule module = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
String fullPath = actionBase.getFullBasePath(table, backend);
actionBase.writeFile(backend, fullPath + "/" + name, content.getBytes());
}
/*******************************************************************************
**
*******************************************************************************/
private S3BackendMetaData defineBackend(QInstance qInstance, String which, String bucketName)
{
QBackendMetaData backendMetaData = new S3BackendMetaData()
.withBucketName(bucketName)
.withBasePath("root")
.withName("backend-" + which);
qInstance.addBackend(backendMetaData);
return (S3BackendMetaData) backendMetaData;
}
/*******************************************************************************
**
*******************************************************************************/
private QTableMetaData defineTable(QInstance qInstance, String which, QBackendMetaData backend, String path, String glob)
{
QTableMetaData qTableMetaData = new QTableMetaData()
.withName("table-" + which)
.withBackendName(backend.getName())
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withBackendDetails(new S3TableBackendDetails()
.withBasePath(path)
.withGlob(glob));
qInstance.addTable(qTableMetaData);
return (qTableMetaData);
}
}

123
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java Normal file
View File

@ -0,0 +1,123 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.sync;
import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for FilesystemSyncProcess
*******************************************************************************/
class FilesystemSyncProcessTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void test() throws Exception
{
TestUtils.cleanInstanceFiles();
QTableMetaData sourceTable = defineTable("source");
QTableMetaData archiveTable = defineTable("archive");
QTableMetaData processingTable = defineTable("processing");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QBackendStepMetaData step = (QBackendStepMetaData) process.getStep(FilesystemSyncStep.STEP_NAME);
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_PROCESSING_TABLE).setDefaultValue(processingTable.getName());
// step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE).setDefaultValue(1);
QInstance qInstance = TestUtils.defineInstance();
qInstance.addTable(sourceTable);
qInstance.addTable(archiveTable);
qInstance.addTable(processingTable);
qInstance.addProcess(process);
///////////////////////////
// write some test files //
///////////////////////////
String basePath = ((FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS)).getBasePath();
writeTestFile(basePath, sourceTable, "1.txt", "x");
writeTestFile(basePath, sourceTable, "2.txt", "x");
// writeTestFile(basePath, sourceTable, "3.txt", "x");
writeTestFile(basePath, archiveTable, "2.txt", "x");
//////////////////////
// run the step //
//////////////////////
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(qInstance);
runBackendStepInput.setStepName(step.getName());
runBackendStepInput.setProcessName(process.getName());
runBackendStepInput.setSession(TestUtils.getMockSession());
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepOutput runBackendStepOutput = runFunctionAction.execute(runBackendStepInput);
// System.out.println(runBackendStepResult);
}
/*******************************************************************************
**
*******************************************************************************/
private void writeTestFile(String basePath, QTableMetaData table, String name, String content) throws IOException
{
String path = ((FilesystemTableBackendDetails) table.getBackendDetails()).getBasePath();
File file = new File(basePath + "/" + path + "/" + name);
FileUtils.writeStringToFile(file, content);
}
/*******************************************************************************
**
*******************************************************************************/
private QTableMetaData defineTable(String subPath)
{
return new QTableMetaData()
.withName("table-" + subPath)
.withBackendName(TestUtils.BACKEND_NAME_LOCAL_FS)
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath(subPath));
}
}

162
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java Normal file
View File

@ -0,0 +1,162 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3;
import cloud.localstack.ServiceName;
import cloud.localstack.awssdkv1.TestUtils;
import cloud.localstack.docker.LocalstackDockerExtension;
import cloud.localstack.docker.annotation.LocalstackDockerProperties;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.ExtendWith;
/*******************************************************************************
** Base class for tests that want to be able to work with localstack s3.
*******************************************************************************/
@ExtendWith(LocalstackDockerExtension.class)
@LocalstackDockerProperties(services = { ServiceName.S3 }, portEdge = "2960", portElasticSearch = "2961")
public class BaseS3Test
{
public static final String BUCKET_NAME = "localstack-test-bucket";
public static final String TEST_FOLDER = "test-files";
public static final String SUB_FOLDER = "sub-folder";
/*******************************************************************************
** Before each unit test, get the test bucket into a known state
*******************************************************************************/
@BeforeEach
public void beforeEach()
{
AmazonS3 amazonS3 = getAmazonS3();
amazonS3.createBucket(BUCKET_NAME);
amazonS3.putObject(BUCKET_NAME, "0.csv", getCSVHeader());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/1.csv", getCSVData1());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2());
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test");
amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
}
/*******************************************************************************
** After each unit test, clean up the bucket
*******************************************************************************/
@AfterEach
public void afterEach()
{
AmazonS3 amazonS3 = getAmazonS3();
if(amazonS3.doesBucketExistV2(BUCKET_NAME))
{
////////////////////////
// todo - paginate... //
////////////////////////
for(S3ObjectSummary objectSummary : amazonS3.listObjectsV2(BUCKET_NAME).getObjectSummaries())
{
amazonS3.deleteObject(BUCKET_NAME, objectSummary.getKey());
}
amazonS3.deleteBucket(BUCKET_NAME);
}
}
/*******************************************************************************
** Access a localstack-configured s3 client.
*******************************************************************************/
protected AmazonS3 getAmazonS3()
{
return (TestUtils.getClientS3());
}
/*******************************************************************************
** Access the S3Utils object, with localstack-configured s3 client.
*******************************************************************************/
protected S3Utils getS3Utils()
{
S3Utils s3Utils = new S3Utils();
s3Utils.setAmazonS3(getAmazonS3());
return (s3Utils);
}
/*******************************************************************************
** Access a string of CSV test data.
*******************************************************************************/
protected String getCSVHeader()
{
return ("""
"id","createDate","modifyDate","firstName","lastName","birthDate","email"
""");
}
/*******************************************************************************
** Access a string of CSV test data.
*******************************************************************************/
protected String getCSVData1()
{
return (getCSVHeader() + """
"1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com"
"2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com"
""");
}
/*******************************************************************************
** Access a string of CSV test data.
*******************************************************************************/
protected String getCSVData2()
{
return (getCSVHeader() + """
"3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com"
"4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com"
"5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com"
""");
}
/*******************************************************************************
** Access a string of CSV test data.
*******************************************************************************/
protected String getCSVData3()
{
return (getCSVHeader() + """
"6","2022-06-20 15:31:02","2022-06-20 15:31:02","Lisa","S","1986-06-06","lisa.s@kingsrook.com"
""");
}
}

53
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleSubclassForTest.java Normal file
View File

@ -0,0 +1,53 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import cloud.localstack.awssdkv1.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
/*******************************************************************************
** Subclass of the S3Backend module, meant for use in unit tests, if/where we
** need to make sure we use the localstack version of the S3 client.
*******************************************************************************/
public class S3BackendModuleSubclassForTest extends S3BackendModule
{
/*******************************************************************************
** Seed the AbstractS3Action with an s3Utils object that has the localstack
** s3 client in it
*******************************************************************************/
@Override
public AbstractBaseFilesystemAction<S3ObjectSummary> getActionBase()
{
AbstractS3Action actionBase = (AbstractS3Action) super.getActionBase();
S3Utils s3Utils = new S3Utils();
s3Utils.setAmazonS3(TestUtils.getClientS3());
actionBase.setS3Utils(s3Utils);
return (actionBase);
}
}

156
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java Normal file
View File

@ -0,0 +1,156 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3;
import java.util.List;
import java.util.UUID;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for S3BackendModule
*******************************************************************************/
public class S3BackendModuleTest extends BaseS3Test
{
private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist";
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDeleteFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
/////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then delete one, then re-list, and assert that we have one fewer //
/////////////////////////////////////////////////////////////////////////////////////////////
List<S3ObjectSummary> s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
S3BackendModule s3BackendModule = new S3BackendModule();
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
actionBase.setS3Utils(getS3Utils());
actionBase.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey());
List<S3ObjectSummary> s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(),
"Should be one fewer file listed after deleting one.");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDeleteFileDoesNotExist() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
// note, we'd like to detect the non-delete, but there's no such info back from aws it appears? //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<S3ObjectSummary> s3ObjectSummariesBeforeDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
S3BackendModule s3BackendModule = new S3BackendModule();
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
actionBase.setS3Utils(getS3Utils());
actionBase.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST);
List<S3ObjectSummary> s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size(), s3ObjectSummariesAfterDelete.size(),
"Should be same number of files after deleting bogus path");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMoveFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
String subPath = TEST_FOLDER + "/" + SUB_FOLDER;
////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files (non-recursively) - then move one into a sub-folder, then re-list, and //
// assert that we have one fewer then list again including sub-folders, and see the changed count //
////////////////////////////////////////////////////////////////////////////////////////////////////
List<S3ObjectSummary> s3ObjectSummariesBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
List<S3ObjectSummary> s3ObjectSummariesInSubFolderBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, subPath, "");
List<S3ObjectSummary> s3ObjectSummariesRecursiveBeforeMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**");
S3BackendModule s3BackendModule = new S3BackendModule();
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
actionBase.setS3Utils(getS3Utils());
String key = s3ObjectSummariesBeforeMove.get(0).getKey();
actionBase.moveFile(qInstance, table, key, key.replaceFirst(TEST_FOLDER, subPath));
List<S3ObjectSummary> s3ObjectSummariesAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "");
List<S3ObjectSummary> s3ObjectSummariesRecursiveAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**");
List<S3ObjectSummary> s3ObjectSummariesInSubFolderAfterMove = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, subPath, "");
Assertions.assertEquals(s3ObjectSummariesBeforeMove.size() - 1, s3ObjectSummariesAfterMove.size(),
"Should be one fewer file in the non-recursive listing after moving one.");
Assertions.assertEquals(s3ObjectSummariesRecursiveBeforeMove.size(), s3ObjectSummariesRecursiveAfterMove.size(),
"Should be same number of files in the recursive listing before and after the move");
Assertions.assertEquals(s3ObjectSummariesInSubFolderBeforeMove.size() + 1, s3ObjectSummariesInSubFolderAfterMove.size(),
"Should be one move file in the sub-folder listing after moving one.");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMoveFileDoesNotExit() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3);
String subPath = TEST_FOLDER + "/" + SUB_FOLDER;
S3BackendModule s3BackendModule = new S3BackendModule();
AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase();
actionBase.setS3Utils(getS3Utils());
Assertions.assertThrows(FilesystemException.class, () ->
actionBase.moveFile(qInstance, table, PATH_THAT_WONT_EXIST, subPath + "/" + UUID.randomUUID())
);
}
}

67
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3ActionTest.java Normal file
View File

@ -0,0 +1,67 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
/*******************************************************************************
** Unit test for AbstractS3Action
*******************************************************************************/
class AbstractS3ActionTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBuildAmazonS3ClientFromBackendMetaData()
{
String regionName = Regions.AP_SOUTHEAST_3.getName();
S3BackendMetaData s3BackendMetaData = new S3BackendMetaData()
.withAccessKey("Not a real access key")
.withSecretKey("Also not a real key")
.withRegion(regionName);
AmazonS3 amazonS3 = new AbstractS3Action().buildAmazonS3ClientFromBackendMetaData(s3BackendMetaData);
assertEquals(regionName, amazonS3.getRegionName());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBuildAmazonS3ClientFromBackendMetaDataWrongType()
{
assertThrows(IllegalArgumentException.class, () ->
{
new AbstractS3Action().buildAmazonS3ClientFromBackendMetaData(new QBackendMetaData());
});
}
}

71
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java Normal file
View File

@ -0,0 +1,71 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/*******************************************************************************
**
*******************************************************************************/
public class S3QueryActionTest extends BaseS3Test
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testQuery1() throws QException
{
QueryInput queryInput = initQueryRequest();
S3QueryAction s3QueryAction = new S3QueryAction();
s3QueryAction.setS3Utils(getS3Utils());
QueryOutput queryOutput = s3QueryAction.execute(queryInput);
Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
Assertions.assertTrue(queryOutput.getRecords().stream()
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseS3Test.TEST_FOLDER)),
"All records should have a full-path in their backend details, matching the test folder name");
}
/*******************************************************************************
**
*******************************************************************************/
private QueryInput initQueryRequest() throws QInstanceValidationException
{
QueryInput queryInput = new QueryInput();
queryInput.setInstance(TestUtils.defineInstance());
queryInput.setTableName(TestUtils.defineS3CSVPersonTable().getName());
return queryInput;
}
}

76
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java Normal file
View File

@ -0,0 +1,76 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for S3BackendMetaData
*******************************************************************************/
class S3BackendMetaDataTest
{
/*******************************************************************************
** Test that an instance can be serialized as expected
*******************************************************************************/
@Test
public void testSerializingToJson() throws QInstanceValidationException
{
TestUtils.resetTestInstanceCounter();
QInstance qInstance = TestUtils.defineInstance();
String json = new QInstanceAdapter().qInstanceToJsonIncludingBackend(qInstance);
System.out.println(JsonUtils.prettyPrint(json));
System.out.println(json);
String expectToContain = """
{"s3":{"bucketName":"localstack-test-bucket","basePath":"test-files","backendType":"s3","name":"s3"}""";
assertTrue(json.contains(expectToContain));
}
/*******************************************************************************
** Test that an instance can be deserialized as expected
*******************************************************************************/
@Test
public void testDeserializingFromJson() throws IOException, QInstanceValidationException
{
QInstanceAdapter qInstanceAdapter = new QInstanceAdapter();
QInstance qInstance = TestUtils.defineInstance();
String json = qInstanceAdapter.qInstanceToJsonIncludingBackend(qInstance);
QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json);
assertThat(deserialized.getBackends()).usingRecursiveComparison()
.isEqualTo(qInstance.getBackends());
}
}

73
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/PathMatcherGlobTest.java Normal file
View File

@ -0,0 +1,73 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
import java.net.URI;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Verification for some of the behavior in the S3Utils - working with PathMatcher
** globs.
*******************************************************************************/
public class PathMatcherGlobTest
{
@Test
public void testPathMatcher() throws Exception
{
/////////////////////////////////////////////////////////////
// note: must start with for both the pattern and the uri //
/////////////////////////////////////////////////////////////
PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/*/acme/*/*.csv");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/1234.csv"))), "Glob should match");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/nj/acme/20220627/1234.csv"))), "Glob should match");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220628/1234.csv"))), "Glob should match");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/12345.csv"))), "Glob should match");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/beta/20220627/1234.csv"))), "Glob should not match (beta vs acme)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///something/stl/acme/20220627/1234.csv"))), "Glob should not match (wrong start path)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/csv"))), "Glob should not match (no file basename)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/1234.CSV"))), "Glob should not match (wrong case extension)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/acme/20220627/extra/1234.csv"))), "Glob should not match (extra dir)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/stl/extra/acme/20220627/1234.csv"))), "Glob should not match (extra dir)");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/extra/stl/acme/20220627/1234.csv"))), "Glob should not match (extra dir)");
pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/**/acme/*/*.csv");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/extra/stl/acme/20220627/1234.csv"))), "Glob should match with extra dir");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/extra/extra2/stl/acme/20220627/1234.csv"))), "Glob should match with 2 extra dirs");
Assertions.assertFalse(pathMatcher.matches(Path.of(URI.create("file:///root/acme/20220627/1234.csv"))), "Glob does not match with no dir for **");
pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/root/**");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///root/1234.csv"))), "Glob should match with extra dir");
pathMatcher = FileSystems.getDefault().getPathMatcher("glob:/*");
Assertions.assertTrue(pathMatcher.matches(Path.of(URI.create("file:///1234.csv"))), "Glob should match");
}
}

84
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java Normal file
View File

@ -0,0 +1,84 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
**
*******************************************************************************/
public class S3UtilsTest extends BaseS3Test
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testListObjectsInBucketAtPath()
{
S3Utils s3Utils = getS3Utils();
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders");
assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders");
assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders");
assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash");
assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, "").size(), "Just in the subfolder non-recursive");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER, "/**").size(), "Just in the subfolder recursive");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER + "//" + SUB_FOLDER, "/**").size(), "Just in the subfolder recursive, multi /");
assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "not-a-real-path/", "").size(), "In a non-existing folder");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s");
assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string");
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testGetObjectAsInputStream() throws IOException
{
S3Utils s3Utils = getS3Utils();
List<S3ObjectSummary> s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", "");
S3ObjectSummary s3ObjectSummary = s3ObjectSummaries.stream().filter(o -> o.getKey().contains("1.csv")).findAny().get();
InputStream inputStream = s3Utils.getObjectAsInputStream(s3ObjectSummary);
String csvFromS3 = IOUtils.toString(inputStream);
assertEquals(getCSVData1(), csvFromS3, "File from S3 should match expected content");
}
}