QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding filesystem writing - used by javalin to store uploaded files; done async, via new base class for actions

Browse Source
This commit is contained in:
darin.kelkhoff
2022-11-17 19:56:35 -06:00
parent 6b2860e303
commit 1d1461deea
21 changed files with 728 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/AbstractQActionBiConsumer.java Normal file
View File

@ -0,0 +1,69 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
/*******************************************************************************
** Base class for QQQ Actions (both framework and application defined) that
** have a signature like a BiConsumer - taking both Input and Output objects as
** parameters, with void output.
*******************************************************************************/
public abstract class AbstractQActionBiConsumer<I extends AbstractActionInput, O extends AbstractActionOutput>
{
/*******************************************************************************
**
*******************************************************************************/
public abstract void execute(I input, O output) throws QException;
/*******************************************************************************
**
*******************************************************************************/
public Future<Void> executeAsync(I input, O output)
{
CompletableFuture<Void> completableFuture = new CompletableFuture<>();
Executors.newCachedThreadPool().submit(() ->
{
try
{
execute(input, output);
completableFuture.complete(null);
}
catch(QException e)
{
completableFuture.completeExceptionally(e);
}
});
return (completableFuture);
}
}

69
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/AbstractQActionFunction.java Normal file
View File

@ -0,0 +1,69 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
/*******************************************************************************
** Base class for QQQ Actions (both framework and application defined) that
** have a signature like a Function - taking an Input object as a parameter,
** and returning an Output object.
*******************************************************************************/
public abstract class AbstractQActionFunction<I extends AbstractActionInput, O extends AbstractActionOutput>
{
/*******************************************************************************
**
*******************************************************************************/
public abstract O execute(I input) throws QException;
/*******************************************************************************
**
*******************************************************************************/
public Future<O> executeAsync(I input)
{
CompletableFuture<O> completableFuture = new CompletableFuture<>();
Executors.newCachedThreadPool().submit(() ->
{
try
{
O output = execute(input);
completableFuture.complete(output);
}
catch(QException e)
{
completableFuture.completeExceptionally(e);
}
});
return (completableFuture);
}
}

4
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.actions.tables;
import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
@ -39,7 +40,7 @@ import org.apache.logging.log4j.Logger;
** Action to insert one or more records.
**
*******************************************************************************/
public class InsertAction
public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOutput>
{
private static final Logger LOG = LogManager.getLogger(InsertAction.class);
@ -48,6 +49,7 @@ public class InsertAction
/*******************************************************************************
**
*******************************************************************************/
@Override
public InsertOutput execute(InsertInput insertInput) throws QException
{
ActionHelper.validateSession(insertInput);

43
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
View File

@ -130,6 +130,8 @@ public class QInstanceValidator
validateApps(qInstance);
validatePossibleValueSources(qInstance);
validateQueuesAndProviders(qInstance);
validateUniqueTopLevelNames(qInstance);
}
catch(Exception e)
{
@ -146,6 +148,45 @@ public class QInstanceValidator
/*******************************************************************************
** there can be some unexpected bad-times if you have a table and process, or
** table and app (etc) with the same name (e.g., in app tree building). So,
** just go ahead and make sure those are all unique.
*******************************************************************************/
private void validateUniqueTopLevelNames(QInstance qInstance)
{
String suffix = " is not unique across tables, processes, and apps (but it needs to be)";
Set<String> usedNames = new HashSet<>();
if(qInstance.getTables() != null)
{
for(String tableName : qInstance.getTables().keySet())
{
assertCondition(!usedNames.contains(tableName), "Table name " + tableName + suffix);
usedNames.add(tableName);
}
}
if(qInstance.getProcesses() != null)
{
for(String processName : qInstance.getProcesses().keySet())
{
assertCondition(!usedNames.contains(processName), "Process name " + processName + suffix);
usedNames.add(processName);
}
}
if(qInstance.getApps() != null)
{
for(String appName : qInstance.getApps().keySet())
{
assertCondition(!usedNames.contains(appName), "App name " + appName + suffix);
usedNames.add(appName);
}
}
}
/*******************************************************************************
**
*******************************************************************************/
@ -311,7 +352,7 @@ public class QInstanceValidator
///////////////////////////////
// validate the record label //
///////////////////////////////
if(table.getRecordLabelFields() != null)
if(table.getRecordLabelFields() != null && table.getFields() != null)
{
for(String recordLabelField : table.getRecordLabelFields())
{

10
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java
View File

@ -406,6 +406,16 @@ public class QRecord implements Serializable
/*******************************************************************************
**
*******************************************************************************/
public byte[] getValueByteArray(String fieldName)
{
return (ValueUtils.getValueAsByteArray(values.get(fieldName)));
}
/*******************************************************************************
** Getter for backendDetails
**

6
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/metadata/Auth0AuthenticationMetaData.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.modules.authentication.metadata;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
@ -32,6 +33,11 @@ public class Auth0AuthenticationMetaData extends QAuthenticationMetaData
{
private String baseUrl;
private String clientId;
////////////////////////////////////////////////////////////////////////////////////////
// keep this secret, on the server - don't let it be serialized and sent to a client! //
////////////////////////////////////////////////////////////////////////////////////////
@JsonIgnore
private String clientSecret;

26
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java
View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.utils;
import java.io.Serializable;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
@ -559,4 +560,29 @@ public class ValueUtils
throw (new QValueException("Value [" + value + "] could not be converted to a LocalTime.", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
public static byte[] getValueAsByteArray(Serializable value)
{
if(value == null)
{
return (null);
}
else if(value instanceof byte[] ba)
{
return (ba);
}
else if(value instanceof String s)
{
return (s.getBytes(StandardCharsets.UTF_8));
}
else
{
throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to ByteArray."));
}
}
}

84
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/QInstanceTest.java Normal file
View File

@ -0,0 +1,84 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
/*******************************************************************************
** Unit test for QInstance
*******************************************************************************/
class QInstanceTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetTablePath() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
GetInput getInput = new GetInput(qInstance);
getInput.setSession(new QSession());
String tablePath = qInstance.getTablePath(getInput, TestUtils.TABLE_NAME_PERSON);
assertEquals("/peopleApp/person", tablePath);
////////////////////////////////////////////////////////////////////////////////////////
// call again (to make sure getting from memoization works - verify w/ breakpoint...) //
////////////////////////////////////////////////////////////////////////////////////////
tablePath = qInstance.getTablePath(getInput, TestUtils.TABLE_NAME_PERSON);
assertEquals("/peopleApp/person", tablePath);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetTablePathNotInAnyApp() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
GetInput getInput = new GetInput(qInstance);
getInput.setSession(new QSession());
String tablePath = qInstance.getTablePath(getInput, "notATable");
assertNull(tablePath);
////////////////////////////////////////////////////////////////////////////////////////
// call again (to make sure getting from memoization works - verify w/ breakpoint...) //
////////////////////////////////////////////////////////////////////////////////////////
tablePath = qInstance.getTablePath(getInput, "notATable");
assertNull(tablePath);
}
}

85
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStepTest.java Normal file
View File

@ -0,0 +1,85 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertOrUpdateStep
*******************************************************************************/
class LoadViaInsertOrUpdateStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.getInstance().reset();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
List<QRecord> existingRecordList = List.of(
new QRecord().withValue("id", 47).withValue("firstName", "Tom")
);
TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), existingRecordList);
List<QRecord> inputRecordList = List.of(
new QRecord().withValue("id", 47).withValue("firstName", "Tim"),
new QRecord().withValue("firstName", "John")
);
RunBackendStepInput input = new RunBackendStepInput(qInstance);
input.setSession(new QSession());
input.setRecords(inputRecordList);
input.addValue(LoadViaInsertOrUpdateStep.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_MEMORY);
RunBackendStepOutput output = new RunBackendStepOutput();
new LoadViaInsertOrUpdateStep().run(input, output);
List<QRecord> qRecords = TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY);
assertEquals(2, qRecords.size());
}
}

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -129,7 +129,7 @@ public class TestUtils
public static final String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive";
public static final String PROCESS_NAME_INCREASE_BIRTHDATE = "increaseBirthdate";
public static final String PROCESS_NAME_ADD_TO_PEOPLES_AGE = "addToPeoplesAge";
public static final String PROCESS_NAME_BASEPULL = "basepullTest";
public static final String PROCESS_NAME_BASEPULL = "basepullTestProcess";
public static final String PROCESS_NAME_RUN_SHAPES_PERSON_REPORT = "runShapesPersonReport";
public static final String TABLE_NAME_PERSON_FILE = "personFile";
public static final String TABLE_NAME_PERSON_MEMORY = "personMemory";