QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-19 05:30:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-781 Initial checkin of filesystem importer meta-data template and process

Browse Source
This commit is contained in:
darin.kelkhoff
2024-01-11 07:46:30 -06:00
parent b64883f34f
commit 624a723b54
8 changed files with 1566 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@ -38,12 +39,15 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed.StreamedETLFilesystemBackendStep;
import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer.FilesystemImporterMetaDataTemplate;
import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer.FilesystemImporterProcessMetaDataBuilder;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
@ -59,16 +63,19 @@ public class TestUtils
public static final String BACKEND_NAME_S3 = "s3";
public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix";
public static final String BACKEND_NAME_MOCK = "mock";
public static final String BACKEND_NAME_MEMORY = "memory";
public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json";
public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv";
public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob";
public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive";
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
public static final String TABLE_NAME_BLOB_S3 = "s3-blob";
public static final String TABLE_NAME_PERSON_MOCK = "person-mock";
public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix";
public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed";
public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed";
public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter";
///////////////////////////////////////////////////////////////////
// shouldn't be accessed directly, as we append a counter to it. //
@ -136,15 +143,30 @@ public class TestUtils
qInstance.addTable(defineLocalFilesystemJSONPersonTable());
qInstance.addTable(defineLocalFilesystemCSVPersonTable());
qInstance.addTable(defineLocalFilesystemBlobTable());
qInstance.addTable(defineLocalFilesystemArchiveTable());
qInstance.addBackend(defineS3Backend());
qInstance.addBackend(defineS3BackendSansPrefix());
qInstance.addTable(defineS3CSVPersonTable());
qInstance.addTable(defineS3BlobTable());
qInstance.addTable(defineS3BlobSansPrefixTable());
qInstance.addBackend(defineMockBackend());
qInstance.addBackend(defineMemoryBackend());
qInstance.addTable(defineMockPersonTable());
qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
String importBaseName = "personImporter";
FilesystemImporterProcessMetaDataBuilder filesystemImporterProcessMetaDataBuilder = (FilesystemImporterProcessMetaDataBuilder) new FilesystemImporterProcessMetaDataBuilder()
.withSourceTableName(TABLE_NAME_PERSON_LOCAL_FS_CSV)
.withFileFormat("csv")
.withArchiveFileEnabled(true)
.withArchiveTableName(TABLE_NAME_ARCHIVE_LOCAL_FS)
.withArchivePath("archive-of/personImporterFiles")
.withName(LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
FilesystemImporterMetaDataTemplate filesystemImporterMetaDataTemplate = new FilesystemImporterMetaDataTemplate(qInstance, importBaseName, BACKEND_NAME_MEMORY, filesystemImporterProcessMetaDataBuilder, table -> table.withAuditRules(QAuditRules.defaultInstanceLevelNone()));
filesystemImporterMetaDataTemplate.addToInstance(qInstance);
return (qInstance);
}
@ -257,6 +279,28 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineLocalFilesystemArchiveTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_ARCHIVE_LOCAL_FS)
.withLabel("Archive")
.withBackendName(defineLocalFilesystemBackend().getName())
.withPrimaryKeyField("fileName")
.withField(new QFieldMetaData("fileName", QFieldType.STRING))
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
.withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath("archive")
.withCardinality(Cardinality.ONE)
.withFileNameFieldName("fileName")
.withContentsFieldName("contents")
);
}
/*******************************************************************************
**
*******************************************************************************/
@ -356,6 +400,18 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QBackendMetaData defineMemoryBackend()
{
return (new QBackendMetaData()
.withBackendType(MemoryBackendModule.class)
.withName(BACKEND_NAME_MEMORY));
}
/*******************************************************************************
**
*******************************************************************************/

114
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStepTest.java Normal file
View File

@ -0,0 +1,114 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
import java.io.File;
import java.time.LocalDateTime;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import org.json.JSONObject;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
/*******************************************************************************
** Unit test for FilesystemImporterStep
*******************************************************************************/
class FilesystemImporterStepTest extends FilesystemActionTest
{
//////////////////////////////////////////////////////////////////////////
// note - we take advantage of the @BeforeEach and @AfterEach to set up //
// and clean up files on disk for this test. //
//////////////////////////////////////////////////////////////////////////
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
public void filesystemBaseAfterEach() throws Exception
{
MemoryRecordStore.getInstance().reset();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
RunProcessInput runProcessInput = new RunProcessInput();
runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
new RunProcessAction().execute(runProcessInput);
String importBaseName = "personImporter";
assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
assertEquals(5, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
QRecord record = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1));
assertEquals(1, record.getValue("importFileId"));
assertEquals("John", record.getValue("firstName"));
assertThat(record.getValue("values")).isInstanceOf(String.class);
JSONObject values = new JSONObject(record.getValueString("values"));
assertEquals("John", values.get("firstName"));
FilesystemBackendMetaData backend = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
String basePath = backend.getBasePath();
System.out.println(basePath);
///////////////////////////////////////////
// make sure 2 archive files got created //
///////////////////////////////////////////
LocalDateTime now = LocalDateTime.now();
File[] files = new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth()).listFiles();
assertNotNull(files);
assertEquals(2, files.length);
}
// todo - test json
// todo - test no files found
// todo - confirm delete happens?
// todo - updates?
}

85
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/ImportRecordPostQueryCustomizerTest.java Normal file
View File

@ -0,0 +1,85 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
import java.time.Instant;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
import org.json.JSONObject;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for ImportRecordPostQueryCustomizer
*******************************************************************************/
class ImportRecordPostQueryCustomizerTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test()
{
Instant createDate = Instant.parse("2024-01-08T20:07:21Z");
List<QRecord> output = new ImportRecordPostQueryCustomizer().apply(List.of(
new QRecord()
.withTableName("personImporterImportRecord")
.withValue("importFileId", 1)
.withValue("unmapped", 2)
.withValue("unstructured", 3)
.withValue("nosqlObject", MapBuilder.of(HashMap::new).with("foo", "bar").with("createDate", createDate).build())
));
assertEquals(1, output.get(0).getValue("importFileId"));
assertEquals(2, output.get(0).getValue("unmapped"));
assertEquals(3, output.get(0).getValue("unstructured"));
assertEquals(Map.of("foo", "bar", "createDate", createDate), output.get(0).getValue("nosqlObject"));
///////////////////////////////////////////////////////////////////////////////////////////
// make sure all un-structured fields get put in the "values" field as a JSON string //
// compare as maps, beacuse JSONObject seems to care about the ordering, which, we don't //
///////////////////////////////////////////////////////////////////////////////////////////
Map<String, Object> expectedMap = new JSONObject("""
{
"unmapped": 2,
"unstructured": 3,
"nosqlObject":
{
"foo": "bar",
"createDate": "%s"
}
}
""".formatted(createDate)).toMap();
Map<String, Object> actualMap = new JSONObject(output.get(0).getValueString("values")).toMap();
assertThat(actualMap).isEqualTo(expectedMap);
}
}