QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding filesystem writing - used by javalin to store uploaded files; done async, via new base class for actions

Browse Source
This commit is contained in:
darin.kelkhoff
2022-11-17 19:56:35 -06:00
parent 6b2860e303
commit 1d1461deea
21 changed files with 728 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
View File

@ -33,6 +33,8 @@ import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -45,6 +47,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.NotImplementedException;
@ -328,4 +331,40 @@ public abstract class AbstractBaseFilesystemAction<FILE>
}
}
/*******************************************************************************
**
*******************************************************************************/
protected InsertOutput executeInsert(InsertInput insertInput) throws QException
{
try
{
InsertOutput output = new InsertOutput();
QTableMetaData table = insertInput.getTable();
QBackendMetaData backend = insertInput.getBackend();
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
if(tableDetails.getCardinality().equals(Cardinality.ONE))
{
for(QRecord record : insertInput.getRecords())
{
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString("fileName"));
writeFile(backend, fullPath, record.getValueByteArray("contents"));
record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
output.addRecord(record);
}
}
else
{
throw (new NotImplementedException("Insert is not implemented for filesystem tables with cardinality: " + tableDetails.getCardinality()));
}
return (output);
}
catch(Exception e)
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
}
}

21
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java
View File

@ -26,13 +26,12 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class FilesystemInsertAction implements InsertInterface
public class FilesystemInsertAction extends AbstractFilesystemAction implements InsertInterface
{
/*******************************************************************************
@ -40,23 +39,7 @@ public class FilesystemInsertAction implements InsertInterface
*******************************************************************************/
public InsertOutput execute(InsertInput insertInput) throws QException
{
throw new NotImplementedException("Filesystem insert not implemented");
/*
try
{
InsertResult rs = new InsertResult();
QTableMetaData table = insertRequest.getTable();
List<QRecord> recordsWithStatus = new ArrayList<>();
rs.setRecords(recordsWithStatus);
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
*/
return (super.executeInsert(insertInput));
}
}

22
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java
View File

@ -26,13 +26,12 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class S3InsertAction implements InsertInterface
public class S3InsertAction extends AbstractS3Action implements InsertInterface
{
/*******************************************************************************
@ -40,24 +39,7 @@ public class S3InsertAction implements InsertInterface
*******************************************************************************/
public InsertOutput execute(InsertInput insertInput) throws QException
{
throw new NotImplementedException("S3 insert not implemented");
/*
try
{
InsertResult rs = new InsertResult();
QTableMetaData table = insertRequest.getTable();
List<QRecord> recordsWithStatus = new ArrayList<>();
rs.setRecords(recordsWithStatus);
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
*/
return (super.executeInsert(insertInput));
}
}

46
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -62,7 +62,9 @@ public class TestUtils
public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json";
public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv";
public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob";
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
public static final String TABLE_NAME_BLOB_S3 = "s3-blob";
public static final String TABLE_NAME_PERSON_MOCK = "person-mock";
public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed";
@ -132,8 +134,10 @@ public class TestUtils
qInstance.addBackend(defineLocalFilesystemBackend());
qInstance.addTable(defineLocalFilesystemJSONPersonTable());
qInstance.addTable(defineLocalFilesystemCSVPersonTable());
qInstance.addTable(defineLocalFilesystemBlobTable());
qInstance.addBackend(defineS3Backend());
qInstance.addTable(defineS3CSVPersonTable());
qInstance.addTable(defineS3BlobTable());
qInstance.addBackend(defineMockBackend());
qInstance.addTable(defineMockPersonTable());
qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
@ -230,6 +234,46 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineLocalFilesystemBlobTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_BLOB_LOCAL_FS)
.withLabel("Blob")
.withBackendName(defineLocalFilesystemBackend().getName())
.withPrimaryKeyField("fileName")
.withField(new QFieldMetaData("fileName", QFieldType.STRING))
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
.withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath("blobs")
.withCardinality(Cardinality.ONE)
);
}
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineS3BlobTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_BLOB_S3)
.withLabel("Blob S3")
.withBackendName(defineS3Backend().getName())
.withPrimaryKeyField("fileName")
.withField(new QFieldMetaData("fileName", QFieldType.STRING))
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
.withBackendDetails(new S3TableBackendDetails()
.withBasePath("blobs")
.withCardinality(Cardinality.ONE)
);
}
/*******************************************************************************
**
*******************************************************************************/
@ -296,7 +340,7 @@ public class TestUtils
{
QProcessMetaData qProcessMetaData = new StreamedETLProcess().defineProcessMetaData();
qProcessMetaData.setName(PROCESS_NAME_STREAMED_ETL);
QBackendStepMetaData backendStep = qProcessMetaData.getBackendStep(StreamedETLProcess.FUNCTION_NAME_ETL);
QBackendStepMetaData backendStep = qProcessMetaData.getBackendStep(StreamedETLProcess.FUNCTION_NAME_ETL);
backendStep.setCode(new QCodeReference(StreamedETLFilesystemBackendStep.class));
backendStep.getInputMetaData().getFieldThrowing(StreamedETLProcess.FIELD_SOURCE_TABLE).setDefaultValue(TABLE_NAME_PERSON_LOCAL_FS_CSV);

50
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertActionTest.java
View File

@ -22,11 +22,24 @@
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.io.File;
import java.io.IOException;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
@ -39,9 +52,40 @@ public class FilesystemInsertActionTest extends FilesystemActionTest
**
*******************************************************************************/
@Test
public void test() throws QException
public void testCardinalityOne() throws QException, IOException
{
assertThrows(NotImplementedException.class, () -> new FilesystemInsertAction().execute(new InsertInput()));
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file1.txt").withValue("contents", "Hello, World")
));
InsertOutput insertOutput = new InsertAction().execute(insertInput);
assertThat(insertOutput.getRecords())
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH))
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
assertEquals("Hello, World", FileUtils.readFileToString(new File(insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCardinalityMany() throws QException, IOException
{
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
insertInput.setRecords(List.of(
new QRecord().withValue("id", "1").withValue("firstName", "Bob")
));
assertThatThrownBy(() -> new InsertAction().execute(insertInput))
.hasRootCauseInstanceOf(NotImplementedException.class);
}
}

56
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertActionTest.java
View File

@ -22,12 +22,25 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import java.io.IOException;
import java.util.List;
import com.amazonaws.services.s3.model.S3Object;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
@ -40,9 +53,46 @@ public class S3InsertActionTest extends BaseS3Test
**
*******************************************************************************/
@Test
public void test() throws QException
public void testCardinalityOne() throws QException, IOException
{
assertThrows(NotImplementedException.class, () -> new S3InsertAction().execute(new InsertInput()));
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
));
S3InsertAction insertAction = new S3InsertAction();
insertAction.setS3Utils(getS3Utils());
InsertOutput insertOutput = insertAction.execute(insertInput);
assertThat(insertOutput.getRecords())
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
String fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
List lines = IOUtils.readLines(object.getObjectContent());
assertEquals("Hi, Bob.", lines.get(0));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCardinalityMany() throws QException, IOException
{
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_S3);
insertInput.setRecords(List.of(
new QRecord().withValue("id", "1").withValue("firstName", "Bob")
));
assertThatThrownBy(() -> new InsertAction().execute(insertInput))
.hasRootCauseInstanceOf(NotImplementedException.class);
}
}