QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-19 05:30:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding filesystem writing - used by javalin to store uploaded files; done async, via new base class for actions

Browse Source
This commit is contained in:
darin.kelkhoff
2022-11-17 19:56:35 -06:00
parent 6b2860e303
commit 1d1461deea
21 changed files with 728 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -62,7 +62,9 @@ public class TestUtils
public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json";
public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv";
public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob";
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
public static final String TABLE_NAME_BLOB_S3 = "s3-blob";
public static final String TABLE_NAME_PERSON_MOCK = "person-mock";
public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed";
@ -132,8 +134,10 @@ public class TestUtils
qInstance.addBackend(defineLocalFilesystemBackend());
qInstance.addTable(defineLocalFilesystemJSONPersonTable());
qInstance.addTable(defineLocalFilesystemCSVPersonTable());
qInstance.addTable(defineLocalFilesystemBlobTable());
qInstance.addBackend(defineS3Backend());
qInstance.addTable(defineS3CSVPersonTable());
qInstance.addTable(defineS3BlobTable());
qInstance.addBackend(defineMockBackend());
qInstance.addTable(defineMockPersonTable());
qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
@ -230,6 +234,46 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineLocalFilesystemBlobTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_BLOB_LOCAL_FS)
.withLabel("Blob")
.withBackendName(defineLocalFilesystemBackend().getName())
.withPrimaryKeyField("fileName")
.withField(new QFieldMetaData("fileName", QFieldType.STRING))
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
.withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath("blobs")
.withCardinality(Cardinality.ONE)
);
}
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineS3BlobTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_BLOB_S3)
.withLabel("Blob S3")
.withBackendName(defineS3Backend().getName())
.withPrimaryKeyField("fileName")
.withField(new QFieldMetaData("fileName", QFieldType.STRING))
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
.withBackendDetails(new S3TableBackendDetails()
.withBasePath("blobs")
.withCardinality(Cardinality.ONE)
);
}
/*******************************************************************************
**
*******************************************************************************/
@ -296,7 +340,7 @@ public class TestUtils
{
QProcessMetaData qProcessMetaData = new StreamedETLProcess().defineProcessMetaData();
qProcessMetaData.setName(PROCESS_NAME_STREAMED_ETL);
QBackendStepMetaData backendStep = qProcessMetaData.getBackendStep(StreamedETLProcess.FUNCTION_NAME_ETL);
QBackendStepMetaData backendStep = qProcessMetaData.getBackendStep(StreamedETLProcess.FUNCTION_NAME_ETL);
backendStep.setCode(new QCodeReference(StreamedETLFilesystemBackendStep.class));
backendStep.getInputMetaData().getFieldThrowing(StreamedETLProcess.FIELD_SOURCE_TABLE).setDefaultValue(TABLE_NAME_PERSON_LOCAL_FS_CSV);

50
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertActionTest.java
View File

@ -22,11 +22,24 @@
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.io.File;
import java.io.IOException;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
@ -39,9 +52,40 @@ public class FilesystemInsertActionTest extends FilesystemActionTest
**
*******************************************************************************/
@Test
public void test() throws QException
public void testCardinalityOne() throws QException, IOException
{
assertThrows(NotImplementedException.class, () -> new FilesystemInsertAction().execute(new InsertInput()));
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file1.txt").withValue("contents", "Hello, World")
));
InsertOutput insertOutput = new InsertAction().execute(insertInput);
assertThat(insertOutput.getRecords())
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH))
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
assertEquals("Hello, World", FileUtils.readFileToString(new File(insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCardinalityMany() throws QException, IOException
{
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
insertInput.setRecords(List.of(
new QRecord().withValue("id", "1").withValue("firstName", "Bob")
));
assertThatThrownBy(() -> new InsertAction().execute(insertInput))
.hasRootCauseInstanceOf(NotImplementedException.class);
}
}

56
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertActionTest.java
View File

@ -22,12 +22,25 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import java.io.IOException;
import java.util.List;
import com.amazonaws.services.s3.model.S3Object;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
@ -40,9 +53,46 @@ public class S3InsertActionTest extends BaseS3Test
**
*******************************************************************************/
@Test
public void test() throws QException
public void testCardinalityOne() throws QException, IOException
{
assertThrows(NotImplementedException.class, () -> new S3InsertAction().execute(new InsertInput()));
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
));
S3InsertAction insertAction = new S3InsertAction();
insertAction.setS3Utils(getS3Utils());
InsertOutput insertOutput = insertAction.execute(insertInput);
assertThat(insertOutput.getRecords())
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
String fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
List lines = IOUtils.readLines(object.getObjectContent());
assertEquals("Hi, Bob.", lines.get(0));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCardinalityMany() throws QException, IOException
{
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_S3);
insertInput.setRecords(List.of(
new QRecord().withValue("id", "1").withValue("firstName", "Bob")
));
assertThatThrownBy(() -> new InsertAction().execute(insertInput))
.hasRootCauseInstanceOf(NotImplementedException.class);
}
}