QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding filesystem writing - used by javalin to store uploaded files; done async, via new base class for actions

Browse Source
This commit is contained in:
darin.kelkhoff
2022-11-17 19:56:35 -06:00
parent 6b2860e303
commit 1d1461deea
21 changed files with 728 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/AbstractQActionBiConsumer.java Normal file
View File

@ -0,0 +1,69 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
/*******************************************************************************
** Base class for QQQ Actions (both framework and application defined) that
** have a signature like a BiConsumer - taking both Input and Output objects as
** parameters, with void output.
*******************************************************************************/
public abstract class AbstractQActionBiConsumer<I extends AbstractActionInput, O extends AbstractActionOutput>
{
/*******************************************************************************
**
*******************************************************************************/
public abstract void execute(I input, O output) throws QException;
/*******************************************************************************
**
*******************************************************************************/
public Future<Void> executeAsync(I input, O output)
{
CompletableFuture<Void> completableFuture = new CompletableFuture<>();
Executors.newCachedThreadPool().submit(() ->
{
try
{
execute(input, output);
completableFuture.complete(null);
}
catch(QException e)
{
completableFuture.completeExceptionally(e);
}
});
return (completableFuture);
}
}

69
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/AbstractQActionFunction.java Normal file
View File

@ -0,0 +1,69 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
/*******************************************************************************
** Base class for QQQ Actions (both framework and application defined) that
** have a signature like a Function - taking an Input object as a parameter,
** and returning an Output object.
*******************************************************************************/
public abstract class AbstractQActionFunction<I extends AbstractActionInput, O extends AbstractActionOutput>
{
/*******************************************************************************
**
*******************************************************************************/
public abstract O execute(I input) throws QException;
/*******************************************************************************
**
*******************************************************************************/
public Future<O> executeAsync(I input)
{
CompletableFuture<O> completableFuture = new CompletableFuture<>();
Executors.newCachedThreadPool().submit(() ->
{
try
{
O output = execute(input);
completableFuture.complete(output);
}
catch(QException e)
{
completableFuture.completeExceptionally(e);
}
});
return (completableFuture);
}
}

4
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.actions.tables;
import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
@ -39,7 +40,7 @@ import org.apache.logging.log4j.Logger;
** Action to insert one or more records.
**
*******************************************************************************/
public class InsertAction
public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOutput>
{
private static final Logger LOG = LogManager.getLogger(InsertAction.class);
@ -48,6 +49,7 @@ public class InsertAction
/*******************************************************************************
**
*******************************************************************************/
@Override
public InsertOutput execute(InsertInput insertInput) throws QException
{
ActionHelper.validateSession(insertInput);

43
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
View File

@ -130,6 +130,8 @@ public class QInstanceValidator
validateApps(qInstance);
validatePossibleValueSources(qInstance);
validateQueuesAndProviders(qInstance);
validateUniqueTopLevelNames(qInstance);
}
catch(Exception e)
{
@ -146,6 +148,45 @@ public class QInstanceValidator
/*******************************************************************************
** there can be some unexpected bad-times if you have a table and process, or
** table and app (etc) with the same name (e.g., in app tree building). So,
** just go ahead and make sure those are all unique.
*******************************************************************************/
private void validateUniqueTopLevelNames(QInstance qInstance)
{
String suffix = " is not unique across tables, processes, and apps (but it needs to be)";
Set<String> usedNames = new HashSet<>();
if(qInstance.getTables() != null)
{
for(String tableName : qInstance.getTables().keySet())
{
assertCondition(!usedNames.contains(tableName), "Table name " + tableName + suffix);
usedNames.add(tableName);
}
}
if(qInstance.getProcesses() != null)
{
for(String processName : qInstance.getProcesses().keySet())
{
assertCondition(!usedNames.contains(processName), "Process name " + processName + suffix);
usedNames.add(processName);
}
}
if(qInstance.getApps() != null)
{
for(String appName : qInstance.getApps().keySet())
{
assertCondition(!usedNames.contains(appName), "App name " + appName + suffix);
usedNames.add(appName);
}
}
}
/*******************************************************************************
**
*******************************************************************************/
@ -311,7 +352,7 @@ public class QInstanceValidator
///////////////////////////////
// validate the record label //
///////////////////////////////
if(table.getRecordLabelFields() != null)
if(table.getRecordLabelFields() != null && table.getFields() != null)
{
for(String recordLabelField : table.getRecordLabelFields())
{

10
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java
View File

@ -406,6 +406,16 @@ public class QRecord implements Serializable
/*******************************************************************************
**
*******************************************************************************/
public byte[] getValueByteArray(String fieldName)
{
return (ValueUtils.getValueAsByteArray(values.get(fieldName)));
}
/*******************************************************************************
** Getter for backendDetails
**

6
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/metadata/Auth0AuthenticationMetaData.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.modules.authentication.metadata;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
@ -32,6 +33,11 @@ public class Auth0AuthenticationMetaData extends QAuthenticationMetaData
{
private String baseUrl;
private String clientId;
////////////////////////////////////////////////////////////////////////////////////////
// keep this secret, on the server - don't let it be serialized and sent to a client! //
////////////////////////////////////////////////////////////////////////////////////////
@JsonIgnore
private String clientSecret;

26
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java
View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.utils;
import java.io.Serializable;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
@ -559,4 +560,29 @@ public class ValueUtils
throw (new QValueException("Value [" + value + "] could not be converted to a LocalTime.", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
public static byte[] getValueAsByteArray(Serializable value)
{
if(value == null)
{
return (null);
}
else if(value instanceof byte[] ba)
{
return (ba);
}
else if(value instanceof String s)
{
return (s.getBytes(StandardCharsets.UTF_8));
}
else
{
throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to ByteArray."));
}
}
}

84
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/QInstanceTest.java Normal file
View File

@ -0,0 +1,84 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
/*******************************************************************************
** Unit test for QInstance
*******************************************************************************/
class QInstanceTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetTablePath() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
GetInput getInput = new GetInput(qInstance);
getInput.setSession(new QSession());
String tablePath = qInstance.getTablePath(getInput, TestUtils.TABLE_NAME_PERSON);
assertEquals("/peopleApp/person", tablePath);
////////////////////////////////////////////////////////////////////////////////////////
// call again (to make sure getting from memoization works - verify w/ breakpoint...) //
////////////////////////////////////////////////////////////////////////////////////////
tablePath = qInstance.getTablePath(getInput, TestUtils.TABLE_NAME_PERSON);
assertEquals("/peopleApp/person", tablePath);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetTablePathNotInAnyApp() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
GetInput getInput = new GetInput(qInstance);
getInput.setSession(new QSession());
String tablePath = qInstance.getTablePath(getInput, "notATable");
assertNull(tablePath);
////////////////////////////////////////////////////////////////////////////////////////
// call again (to make sure getting from memoization works - verify w/ breakpoint...) //
////////////////////////////////////////////////////////////////////////////////////////
tablePath = qInstance.getTablePath(getInput, "notATable");
assertNull(tablePath);
}
}

85
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStepTest.java Normal file
View File

@ -0,0 +1,85 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertOrUpdateStep
*******************************************************************************/
class LoadViaInsertOrUpdateStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.getInstance().reset();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
List<QRecord> existingRecordList = List.of(
new QRecord().withValue("id", 47).withValue("firstName", "Tom")
);
TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), existingRecordList);
List<QRecord> inputRecordList = List.of(
new QRecord().withValue("id", 47).withValue("firstName", "Tim"),
new QRecord().withValue("firstName", "John")
);
RunBackendStepInput input = new RunBackendStepInput(qInstance);
input.setSession(new QSession());
input.setRecords(inputRecordList);
input.addValue(LoadViaInsertOrUpdateStep.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_MEMORY);
RunBackendStepOutput output = new RunBackendStepOutput();
new LoadViaInsertOrUpdateStep().run(input, output);
List<QRecord> qRecords = TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY);
assertEquals(2, qRecords.size());
}
}

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -129,7 +129,7 @@ public class TestUtils
public static final String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive";
public static final String PROCESS_NAME_INCREASE_BIRTHDATE = "increaseBirthdate";
public static final String PROCESS_NAME_ADD_TO_PEOPLES_AGE = "addToPeoplesAge";
public static final String PROCESS_NAME_BASEPULL = "basepullTest";
public static final String PROCESS_NAME_BASEPULL = "basepullTestProcess";
public static final String PROCESS_NAME_RUN_SHAPES_PERSON_REPORT = "runShapesPersonReport";
public static final String TABLE_NAME_PERSON_FILE = "personFile";
public static final String TABLE_NAME_PERSON_MEMORY = "personMemory";

39
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
View File

@ -33,6 +33,8 @@ import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -45,6 +47,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.NotImplementedException;
@ -328,4 +331,40 @@ public abstract class AbstractBaseFilesystemAction<FILE>
}
}
/*******************************************************************************
**
*******************************************************************************/
protected InsertOutput executeInsert(InsertInput insertInput) throws QException
{
try
{
InsertOutput output = new InsertOutput();
QTableMetaData table = insertInput.getTable();
QBackendMetaData backend = insertInput.getBackend();
AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
if(tableDetails.getCardinality().equals(Cardinality.ONE))
{
for(QRecord record : insertInput.getRecords())
{
String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString("fileName"));
writeFile(backend, fullPath, record.getValueByteArray("contents"));
record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
output.addRecord(record);
}
}
else
{
throw (new NotImplementedException("Insert is not implemented for filesystem tables with cardinality: " + tableDetails.getCardinality()));
}
return (output);
}
catch(Exception e)
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
}
}

21
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java
View File

@ -26,13 +26,12 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class FilesystemInsertAction implements InsertInterface
public class FilesystemInsertAction extends AbstractFilesystemAction implements InsertInterface
{
/*******************************************************************************
@ -40,23 +39,7 @@ public class FilesystemInsertAction implements InsertInterface
*******************************************************************************/
public InsertOutput execute(InsertInput insertInput) throws QException
{
throw new NotImplementedException("Filesystem insert not implemented");
/*
try
{
InsertResult rs = new InsertResult();
QTableMetaData table = insertRequest.getTable();
List<QRecord> recordsWithStatus = new ArrayList<>();
rs.setRecords(recordsWithStatus);
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
*/
return (super.executeInsert(insertInput));
}
}

22
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java
View File

@ -26,13 +26,12 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
**
*******************************************************************************/
public class S3InsertAction implements InsertInterface
public class S3InsertAction extends AbstractS3Action implements InsertInterface
{
/*******************************************************************************
@ -40,24 +39,7 @@ public class S3InsertAction implements InsertInterface
*******************************************************************************/
public InsertOutput execute(InsertInput insertInput) throws QException
{
throw new NotImplementedException("S3 insert not implemented");
/*
try
{
InsertResult rs = new InsertResult();
QTableMetaData table = insertRequest.getTable();
List<QRecord> recordsWithStatus = new ArrayList<>();
rs.setRecords(recordsWithStatus);
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
*/
return (super.executeInsert(insertInput));
}
}

46
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -62,7 +62,9 @@ public class TestUtils
public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json";
public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv";
public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob";
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
public static final String TABLE_NAME_BLOB_S3 = "s3-blob";
public static final String TABLE_NAME_PERSON_MOCK = "person-mock";
public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed";
@ -132,8 +134,10 @@ public class TestUtils
qInstance.addBackend(defineLocalFilesystemBackend());
qInstance.addTable(defineLocalFilesystemJSONPersonTable());
qInstance.addTable(defineLocalFilesystemCSVPersonTable());
qInstance.addTable(defineLocalFilesystemBlobTable());
qInstance.addBackend(defineS3Backend());
qInstance.addTable(defineS3CSVPersonTable());
qInstance.addTable(defineS3BlobTable());
qInstance.addBackend(defineMockBackend());
qInstance.addTable(defineMockPersonTable());
qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
@ -230,6 +234,46 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineLocalFilesystemBlobTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_BLOB_LOCAL_FS)
.withLabel("Blob")
.withBackendName(defineLocalFilesystemBackend().getName())
.withPrimaryKeyField("fileName")
.withField(new QFieldMetaData("fileName", QFieldType.STRING))
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
.withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath("blobs")
.withCardinality(Cardinality.ONE)
);
}
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineS3BlobTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_BLOB_S3)
.withLabel("Blob S3")
.withBackendName(defineS3Backend().getName())
.withPrimaryKeyField("fileName")
.withField(new QFieldMetaData("fileName", QFieldType.STRING))
.withField(new QFieldMetaData("contents", QFieldType.BLOB))
.withBackendDetails(new S3TableBackendDetails()
.withBasePath("blobs")
.withCardinality(Cardinality.ONE)
);
}
/*******************************************************************************
**
*******************************************************************************/
@ -296,7 +340,7 @@ public class TestUtils
{
QProcessMetaData qProcessMetaData = new StreamedETLProcess().defineProcessMetaData();
qProcessMetaData.setName(PROCESS_NAME_STREAMED_ETL);
QBackendStepMetaData backendStep = qProcessMetaData.getBackendStep(StreamedETLProcess.FUNCTION_NAME_ETL);
QBackendStepMetaData backendStep = qProcessMetaData.getBackendStep(StreamedETLProcess.FUNCTION_NAME_ETL);
backendStep.setCode(new QCodeReference(StreamedETLFilesystemBackendStep.class));
backendStep.getInputMetaData().getFieldThrowing(StreamedETLProcess.FIELD_SOURCE_TABLE).setDefaultValue(TABLE_NAME_PERSON_LOCAL_FS_CSV);

50
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertActionTest.java
View File

@ -22,11 +22,24 @@
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.io.File;
import java.io.IOException;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
@ -39,9 +52,40 @@ public class FilesystemInsertActionTest extends FilesystemActionTest
**
*******************************************************************************/
@Test
public void test() throws QException
public void testCardinalityOne() throws QException, IOException
{
assertThrows(NotImplementedException.class, () -> new FilesystemInsertAction().execute(new InsertInput()));
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file1.txt").withValue("contents", "Hello, World")
));
InsertOutput insertOutput = new InsertAction().execute(insertInput);
assertThat(insertOutput.getRecords())
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH))
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
assertEquals("Hello, World", FileUtils.readFileToString(new File(insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCardinalityMany() throws QException, IOException
{
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
insertInput.setRecords(List.of(
new QRecord().withValue("id", "1").withValue("firstName", "Bob")
));
assertThatThrownBy(() -> new InsertAction().execute(insertInput))
.hasRootCauseInstanceOf(NotImplementedException.class);
}
}

56
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertActionTest.java
View File

@ -22,12 +22,25 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import java.io.IOException;
import java.util.List;
import com.amazonaws.services.s3.model.S3Object;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.NotImplementedException;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
@ -40,9 +53,46 @@ public class S3InsertActionTest extends BaseS3Test
**
*******************************************************************************/
@Test
public void test() throws QException
public void testCardinalityOne() throws QException, IOException
{
assertThrows(NotImplementedException.class, () -> new S3InsertAction().execute(new InsertInput()));
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
));
S3InsertAction insertAction = new S3InsertAction();
insertAction.setS3Utils(getS3Utils());
InsertOutput insertOutput = insertAction.execute(insertInput);
assertThat(insertOutput.getRecords())
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
String fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
List lines = IOUtils.readLines(object.getObjectContent());
assertEquals("Hi, Bob.", lines.get(0));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testCardinalityMany() throws QException, IOException
{
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_S3);
insertInput.setRecords(List.of(
new QRecord().withValue("id", "1").withValue("firstName", "Bob")
));
assertThatThrownBy(() -> new InsertAction().execute(insertInput))
.hasRootCauseInstanceOf(NotImplementedException.class);
}
}

2
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java
View File

@ -83,7 +83,7 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
rs.setRecords(outputRecords);
/////////////////////////////////////////////////////////////////////////////////////////////
// we want to do batch updates. But, since we only update the columns columns that //
// we want to do batch updates. But, since we only update the columns that //
// are present in each record, it means we may have different update SQL for each //
// record. So, we will first "hash" up the records by their list of fields being updated. //
/////////////////////////////////////////////////////////////////////////////////////////////

29
qqq-dev-tools/bin/createTableToEntityFields.groovy
View File

@ -12,6 +12,7 @@ boolean writeTableMetaData = args.length > 2 ? args[2] : false;
def reader = new BufferedReader(new InputStreamReader(System.in))
String line
String allFieldNames = ""
String dataFieldNames = ""
List<String> fieldNameList = new ArrayList<>();
List<String> fieldTypeList = new ArrayList<>();
@ -64,6 +65,15 @@ while((line = reader.readLine()) != null)
""".formatted(attributes, fieldType, fieldName))
allFieldNames += '"' + fieldName + '",'
if(!fieldName.equals("id") && !fieldName.equals("createDate") && !fieldName.equals("modifyDate"))
{
dataFieldNames += '"' + fieldName + '",'
}
}
if(dataFieldNames.length() > 0)
{
dataFieldNames = dataFieldNames.substring(0, dataFieldNames.length() - 1);
}
if(writeWholeClass)
@ -162,22 +172,35 @@ if(writeTableMetaData)
.withRecordLabelFields("TODO")
.withBackendName(TODO)
.withPrimaryKeyField("id")
.withFieldsFromEntity({className}.class)
.withBackendDetails(new RDBMSTableBackendDetails()
.withTableName("{tableName}")
);
)
.withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id")))
.withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of({dataFieldNames})))
.withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
QInstanceEnricher.setInferredFieldBackendNames(qTableMetaData);
return (qTableMetaData);
}
"""
.replaceAll("\\{className}", className)
.replaceAll("\\{tableName}", tableName)
.replaceAll("\\{dataFieldNames}", dataFieldNames)
);
}
println(output);
println()
println("All field names (e.g., for sections):")
println(allFieldNames);
if(!writeTableMetaData)
{
println("All field names (e.g., for sections):")
allFieldNames = allFieldNames.substring(0, allFieldNames.length() - 1);
println(allFieldNames);
}
println();

24
qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
View File

@ -123,7 +123,8 @@ public class QJavalinImplementation
private static final int SESSION_COOKIE_AGE = 60 * 60 * 24;
private static final String SESSION_ID_COOKIE_NAME = "sessionId";
static QInstance qInstance;
static QInstance qInstance;
static QJavalinMetaData javalinMetaData = new QJavalinMetaData();
private static Supplier<QInstance> qInstanceHotSwapSupplier;
private static long lastQInstanceHotSwapMillis;
@ -1063,4 +1064,25 @@ public class QJavalinImplementation
QJavalinImplementation.qInstanceHotSwapSupplier = qInstanceHotSwapSupplier;
}
/*******************************************************************************
** Getter for javalinMetaData
**
*******************************************************************************/
public QJavalinMetaData getJavalinMetaData()
{
return javalinMetaData;
}
/*******************************************************************************
** Setter for javalinMetaData
**
*******************************************************************************/
public void setJavalinMetaData(QJavalinMetaData javalinMetaData)
{
QJavalinImplementation.javalinMetaData = javalinMetaData;
}
}

66
qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinMetaData.java Normal file
View File

@ -0,0 +1,66 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.javalin;
/*******************************************************************************
** MetaData specific to a QQQ Javalin server.
*******************************************************************************/
public class QJavalinMetaData
{
private String uploadedFileArchiveTableName;
/*******************************************************************************
** Getter for uploadedFileArchiveTableName
**
*******************************************************************************/
public String getUploadedFileArchiveTableName()
{
return uploadedFileArchiveTableName;
}
/*******************************************************************************
** Setter for uploadedFileArchiveTableName
**
*******************************************************************************/
public void setUploadedFileArchiveTableName(String uploadedFileArchiveTableName)
{
this.uploadedFileArchiveTableName = uploadedFileArchiveTableName;
}
/*******************************************************************************
** Fluent setter for uploadedFileArchiveTableName
**
*******************************************************************************/
public QJavalinMetaData withUploadedFileArchiveTableName(String uploadedFileArchiveTableName)
{
this.uploadedFileArchiveTableName = uploadedFileArchiveTableName;
return (this);
}
}

29
qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java
View File

@ -22,10 +22,12 @@
package com.kingsrook.qqq.backend.javalin;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Serializable;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@ -42,6 +44,8 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
import com.kingsrook.qqq.backend.core.actions.async.JobGoingAsyncException;
import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
@ -49,6 +53,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@ -290,6 +295,8 @@ public class QJavalinProcessHandler
TempFileStateProvider.getInstance().put(key, qUploadedFile);
LOG.info("Stored uploaded file in TempFileStateProvider under key: " + key);
runProcessInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key);
archiveUploadedFile(runProcessInput, qUploadedFile);
}
/////////////////////////////////////////////////////////////
@ -319,6 +326,28 @@ public class QJavalinProcessHandler
/*******************************************************************************
**
*******************************************************************************/
private static void archiveUploadedFile(RunProcessInput runProcessInput, QUploadedFile qUploadedFile)
{
String fileName = new QValueFormatter().formatDate(LocalDate.now())
+ File.separator + runProcessInput.getProcessName()
+ File.separator + qUploadedFile.getFilename();
InsertInput insertInput = new InsertInput(QJavalinImplementation.qInstance);
insertInput.setSession(runProcessInput.getSession());
insertInput.setTableName(QJavalinImplementation.javalinMetaData.getUploadedFileArchiveTableName());
insertInput.setRecords(List.of(new QRecord()
.withValue("fileName", fileName)
.withValue("contents", qUploadedFile.getBytes())
));
new InsertAction().executeAsync(insertInput);
}
/*******************************************************************************
**
*******************************************************************************/