From 35c079e1dddf7e49bdc0d6684fb31bfe5cd3ffc5 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 10 May 2024 12:07:21 -0500 Subject: [PATCH 01/13] Set a default time zone (UTC) for tests (we have at least one that likes this) --- .../test/java/com/kingsrook/qqq/backend/core/BaseTest.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java index dbe4164c..be72d887 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java @@ -22,6 +22,8 @@ package com.kingsrook.qqq.backend.core; +import java.time.ZoneId; +import java.util.TimeZone; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -42,7 +44,10 @@ public class BaseTest public static final String DEFAULT_USER_ID = "001"; - + static + { + TimeZone.setDefault(TimeZone.getTimeZone(ZoneId.of("UTC"))); + } /******************************************************************************* ** From 00b55c583ed0cc29a9dd0b1f39b1385819fe1887 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 10 May 2024 12:08:07 -0500 Subject: [PATCH 02/13] Fix generic on withRecordEntity be '? extends' --- .../backend/core/model/actions/tables/insert/InsertInput.java | 2 +- .../backend/core/model/actions/tables/update/UpdateInput.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java index 4e750818..1753f61f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java @@ -112,7 +112,7 @@ public class InsertInput extends AbstractTableActionInput /******************************************************************************* ** *******************************************************************************/ - public InsertInput withRecordEntities(List recordEntityList) + public InsertInput withRecordEntities(List recordEntityList) { for(QRecordEntity recordEntity : CollectionUtils.nonNullList(recordEntityList)) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java index 7f81a3ae..767b9ee5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java @@ -120,7 +120,7 @@ public class UpdateInput extends AbstractTableActionInput /******************************************************************************* ** *******************************************************************************/ - public UpdateInput withRecordEntities(List recordEntityList) + public UpdateInput withRecordEntities(List recordEntityList) { for(QRecordEntity recordEntity : CollectionUtils.nonNullList(recordEntityList)) { From f5f02a223459ec26ce236477deaf66fcb7ca8479 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 10 May 2024 12:19:49 -0500 Subject: [PATCH 03/13] Add maxPathLength performance workaround --- .../core/actions/metadata/JoinGraph.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java index fc597ed7..52619b36 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; import java.util.TreeSet; +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -41,9 +42,19 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; *******************************************************************************/ public class JoinGraph { - private Set edges = new HashSet<>(); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // as an instance grows, with the number of joins (say, more than 50?), especially as they may have a lot of connections, // + // it can become very very slow to process a full join graph (e.g., 10 seconds, maybe much worse, per Big-O...) // + // also, it's not frequently useful to look at a join path that's more than a handful of tables long. // + // thus - this property exists - to limit the max length of a join path. Keeping it small keeps instance enrichment // + // and validation reasonably performant, at the possible cost of, some join-path that's longer than this limit may not // + // be found - but - chances are, you don't want some 12-element join path to be used anyway, thus, this makes sense. // + // but - it can be adjusted, per system property or ENV var. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + private int maxPathLength = new QMetaDataVariableInterpreter().getIntegerFromPropertyOrEnvironment("qqq.instance.joinGraph.maxPathLength", "QQQ_INSTANCE_JOIN_GRAPH_MAX_PATH_LENGTH", 3); + /******************************************************************************* @@ -303,6 +314,13 @@ public class JoinGraph if(otherTableName != null) { + if(newPath.size() > maxPathLength) + { + //////////////////////////////////////////////////////////////// + // performance hack. see comment at maxPathLength definition // + //////////////////////////////////////////////////////////////// + continue; + } JoinConnectionList newConnectionList = connectionList.copy(); JoinConnection joinConnection = new JoinConnection(otherTableName, edge.joinName); From 0d3f886e5a3c0a3fdcab327a33d86ee1af8de401 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 10 May 2024 12:20:03 -0500 Subject: [PATCH 04/13] Add overview & basic properties --- docs/metaData/Joins.adoc | 51 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/docs/metaData/Joins.adoc b/docs/metaData/Joins.adoc index 4ce09ea3..3d4adee7 100644 --- a/docs/metaData/Joins.adoc +++ b/docs/metaData/Joins.adoc @@ -2,16 +2,57 @@ == Joins include::../variables.adoc[] -#TODO# +A `QJoinMetaData` is a meta-data object that tells QQQ, essentially “it is possible for these 2 tables to join, here’s how to do it”. + +Joins can be used then, in an application, in a number of possible ways: + +* In a {link-table}, we can specify joins to be “exposed”, e.g., made available to users on a query screen +* Also in a Table, as part of an “Association”, which sets up one table as a “parent” of another, +such that you can store (and fetch) the child-records at the same time as the parent +** A common use-case here may be an order & lineItem table - +such that QQQ can generate an API uses to allow you to post an order and its lines in a single request, and they get stored all together +* In defining the security field (record lock) on a table, +sometimes, it isn’t a field directly on the table, but instead comes from a joined table (possibly even more than just 1 table away). +** For example, maybe a lineItem table, doesn't have a clientId, but needs secured by that field +- so its recordLock can specify a “joinNameChain” that describes how to get from lineItem to order.clientId +* The `QueryAction` can take (through its QueryInput object) zero or more QueryJoin objects, +which must make a reference (implicitly or explicitly) to a QJoinMetaData. +See the section on <> for more details. === QJoinMetaData Joins are defined in a QQQ Instance in a `*QJoinMetaData*` object. -#TODO# +In this object, we have the concept of a "leftTable" and a "rightTable". +There isn't generally anything special about which table is on the "left" and which is on the "right". +But the remaining pieces of the QJoinMetaData do all need to line-up with these sides. + +For example: + +* The Type (one-to-one, one-to-many, many-to-one) - where the leftTable comes first, and rightTable comes second +(e.g., a one-to-many means 1-row in leftTable has many-rows in rightTable associated with it) +* In a JoinOn object, the 1st field name given is from the leftTable; +the second fieldName from the rightTable. *QJoinMetaData Properties:* -* `name` - *String, Required* - Unique name for the join within the QQQ Instance. #todo infererences or conventions?# - -#TODO# +* `name` - *String, Required* - Unique name for the join within the QQQ Instance. +** One convention is to name joins based on (leftTable + "Join" + rightTable). +** If you do not wish to define join names yourself, the method `withInferredName()` +can be called (which defers to +`public static String makeInferredJoinName(String leftTable, String rightTable)`), +to create a name for the join following the (leftTable + "Join" + rightTable) convention. +* `leftTable` - *String, Required* - Name of a {link-table} in the {link-instance}. +* `rightTable` - *String, Required* - Name of a {link-table} in the {link-instance}. +* `type` - *enum, Required* - cardinality between the two tables in the join. +** e.g., `ONE_TO_ONE`, `ONE_TO_MANY` (indicating 1 record in the left table may join +to many records in the right table), or `MANY_TO_ONE` (vice-versa). +** Note that there is no MANY_TO_MANY option, as a many-to-many is built as multiple QJoinMetaData's +going through the intermediary (intersection) table. +* `joinOns` - *List, Required* - fields used to join the tables. +Note: In the 2-arg JoinOn constructor, the leftTable's field comes first. +Alternatively, the no-arg constructor can be used along with `.withLeftField().withRightField()` +* `orderBys` - *List* - Optional list of order-by objects, +used in some framework-generated queries using the join. +The field names are assumed to come from the rightTable. +#TODO# what else do we need here? From 81a5d868b626a10f24718b772c862bf4d7f0d94c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 10 May 2024 12:20:46 -0500 Subject: [PATCH 05/13] Add a non-null filter to avoid an NPE if a null logPair ever gets in --- .../java/com/kingsrook/qqq/backend/core/logging/LogPair.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/LogPair.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/LogPair.java index fd654a0f..6a79a874 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/LogPair.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/LogPair.java @@ -79,7 +79,7 @@ public class LogPair } else if(value instanceof LogPair[] subLogPairs) { - String subLogPairsString = Arrays.stream(subLogPairs).map(LogPair::toString).collect(Collectors.joining(",")); + String subLogPairsString = Arrays.stream(subLogPairs).filter(Objects::nonNull).map(LogPair::toString).collect(Collectors.joining(",")); valueString = '{' + subLogPairsString + '}'; } else if(value instanceof UnsafeSupplier us) From 196488ad6e4f815287efcfdee0038fd31f3a77de Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 10 May 2024 12:24:29 -0500 Subject: [PATCH 06/13] stash in a static field, the list set of topLevel classes (performance gain for setups where this class is called multiple times (along with clearing that cache during a javalin instance hotswap) --- .../metadata/MetaDataProducerHelper.java | 32 +++++++++++++++++-- .../javalin/QJavalinImplementation.java | 12 +++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java index 06a35280..e8165a41 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java @@ -30,6 +30,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import com.google.common.collect.ImmutableSet; import com.google.common.reflect.ClassPath; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; @@ -50,6 +51,8 @@ public class MetaDataProducerHelper private static Map, Integer> comparatorValuesByType = new HashMap<>(); private static Integer defaultComparatorValue; + private static ImmutableSet topLevelClasses; + static { //////////////////////////////////////////////////////////////////////////////////////// @@ -70,8 +73,6 @@ public class MetaDataProducerHelper comparatorValuesByType.put(QAppMetaData.class, 23); } - - /******************************************************************************* ** Recursively find all classes in the given package, that implement MetaDataProducerInterface ** run them, and add their output to the given qInstance. @@ -186,7 +187,7 @@ public class MetaDataProducerHelper List> classes = new ArrayList<>(); ClassLoader loader = Thread.currentThread().getContextClassLoader(); - for(ClassPath.ClassInfo info : ClassPath.from(loader).getTopLevelClasses()) + for(ClassPath.ClassInfo info : getTopLevelClasses(loader)) { if(info.getName().startsWith(packageName)) { @@ -197,4 +198,29 @@ public class MetaDataProducerHelper return (classes); } + + + /******************************************************************************* + ** + *******************************************************************************/ + private static ImmutableSet getTopLevelClasses(ClassLoader loader) throws IOException + { + if(topLevelClasses == null) + { + topLevelClasses = ClassPath.from(loader).getTopLevelClasses(); + } + + return (topLevelClasses); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void clearTopLevelClassCache() + { + topLevelClasses = null; + } + } diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 86a6ed08..276a8733 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -103,6 +103,7 @@ import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSo import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; @@ -283,6 +284,14 @@ public class QJavalinImplementation try { + //////////////////////////////////////////////////////////////////////////////// + // clear the cache of classes in this class, so that new classes can be found // + //////////////////////////////////////////////////////////////////////////////// + MetaDataProducerHelper.clearTopLevelClassCache(); + + ///////////////////////////////////////////////// + // try to get a new instance from the supplier // + ///////////////////////////////////////////////// QInstance newQInstance = qInstanceHotSwapSupplier.get(); if(newQInstance == null) { @@ -290,6 +299,9 @@ public class QJavalinImplementation return; } + /////////////////////////////////////////////////////////////////////////////////// + // validate the instance, and only if it passes, then set it in our static field // + /////////////////////////////////////////////////////////////////////////////////// new QInstanceValidator().validate(newQInstance); QJavalinImplementation.qInstance = newQInstance; LOG.info("Swapped qInstance"); From c9e9d62098cc230b0df8a72007d27160ee708295 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 10 May 2024 12:42:07 -0500 Subject: [PATCH 07/13] Add warnAndThrow and errorAndThrow methods, to slightly simplify catch-blocks that want to do both of those things; also add variant data in session portion of log, when available --- .../qqq/backend/core/logging/QLogger.java | 60 ++++++++++++++++++- .../qqq/backend/core/logging/QLoggerTest.java | 45 ++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java index a00dfb0d..42bfe93d 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.logging; +import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -147,6 +148,28 @@ public class QLogger + /******************************************************************************* + ** + *******************************************************************************/ + public T warnAndThrow(T t, LogPair... logPairs) throws T + { + warn(t.getMessage(), t, logPairs); + throw (t); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public T errorAndThrow(T t, LogPair... logPairs) throws T + { + error(t.getMessage(), t, logPairs); + throw (t); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -595,7 +618,10 @@ public class QLogger { user = session.getUser().getIdReference(); } - sessionLogPair = logPair("session", logPair("id", session.getUuid()), logPair("user", user)); + + LogPair variantsLogPair = getVariantsLogPair(session); + + sessionLogPair = logPair("session", logPair("id", session.getUuid()), logPair("user", user), variantsLogPair); } try @@ -615,6 +641,38 @@ public class QLogger + /******************************************************************************* + ** + *******************************************************************************/ + private static LogPair getVariantsLogPair(QSession session) + { + LogPair variantsLogPair = null; + try + { + if(session.getBackendVariants() != null) + { + LogPair[] variants = new LogPair[session.getBackendVariants().size()]; + + int i = 0; + for(Map.Entry entry : session.getBackendVariants().entrySet()) + { + variants[i] = new LogPair(entry.getKey(), entry.getValue()); + } + + variantsLogPair = new LogPair("variants", variants); + } + } + catch(Exception e) + { + //////////////// + // leave null // + //////////////// + } + return variantsLogPair; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QLoggerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QLoggerTest.java index 1c28982c..de4b60eb 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QLoggerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QLoggerTest.java @@ -43,6 +43,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /******************************************************************************* @@ -67,6 +68,50 @@ class QLoggerTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testLogAndThrowMethods() throws QException + { + try + { + LOG.info("Some info"); + LOG.warnAndThrow(new QException("Something failed"), new LogPair("something", 1)); + } + catch(Exception e) + { + ////////////// + // ok, done // + ////////////// + } + + assertThatThrownBy(() -> + { + try + { + methodThatThrows(); + } + catch(Exception e) + { + throw LOG.errorAndThrow(new QException("I caught, now i errorAndThrow", e), new LogPair("iLove", "logPairs")); + } + } + ).isInstanceOf(QException.class).hasMessageContaining("I caught").rootCause().hasMessageContaining("See, I throw"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void methodThatThrows() throws QException + { + throw (new QException("See, I throw")); + } + + + /******************************************************************************* ** *******************************************************************************/ From f009d50631b7c1c515b4cf3366c239d3cd4dc64d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 10 May 2024 12:51:20 -0500 Subject: [PATCH 08/13] CE-1180 add option to setPrimaryKeyInInsertedRecords (on the INPUT records) --- .../core/actions/tables/ReplaceAction.java | 11 ++++++ .../actions/tables/replace/ReplaceInput.java | 36 +++++++++++++++++-- .../actions/tables/ReplaceActionTest.java | 25 +++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java index 04177cdb..ca2d1413 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java @@ -157,6 +157,17 @@ public class ReplaceAction extends AbstractQActionFunction records; private QQueryFilter filter; - private boolean performDeletes = true; - private boolean allowNullKeyValuesToEqual = false; + private boolean performDeletes = true; + private boolean allowNullKeyValuesToEqual = false; + private boolean setPrimaryKeyInInsertedRecords = false; private boolean omitDmlAudit = false; @@ -271,4 +272,35 @@ public class ReplaceInput extends AbstractTableActionInput return (this); } + + + /******************************************************************************* + ** Getter for setPrimaryKeyInInsertedRecords + *******************************************************************************/ + public boolean getSetPrimaryKeyInInsertedRecords() + { + return (this.setPrimaryKeyInInsertedRecords); + } + + + + /******************************************************************************* + ** Setter for setPrimaryKeyInInsertedRecords + *******************************************************************************/ + public void setSetPrimaryKeyInInsertedRecords(boolean setPrimaryKeyInInsertedRecords) + { + this.setPrimaryKeyInInsertedRecords = setPrimaryKeyInInsertedRecords; + } + + + + /******************************************************************************* + ** Fluent setter for setPrimaryKeyInInsertedRecords + *******************************************************************************/ + public ReplaceInput withSetPrimaryKeyInInsertedRecords(boolean setPrimaryKeyInInsertedRecords) + { + this.setPrimaryKeyInInsertedRecords = setPrimaryKeyInInsertedRecords; + return (this); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java index da57c703..9eb5dc57 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.tables; import java.util.List; import java.util.Map; +import java.util.Optional; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; @@ -38,8 +39,11 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* @@ -81,12 +85,27 @@ class ReplaceActionTest extends BaseTest replaceInput.setOmitDmlAudit(true); replaceInput.setRecords(newPeople); replaceInput.setFilter(null); + replaceInput.setSetPrimaryKeyInInsertedRecords(false); ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput); assertEquals(1, replaceOutput.getInsertOutput().getRecords().size()); assertEquals(1, replaceOutput.getUpdateOutput().getRecords().size()); assertEquals(1, replaceOutput.getDeleteOutput().getDeletedRecordCount()); + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // due to false for SetPrimaryKeyInInsertedRecords, make sure primary keys aren't on the records that got inserted // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Optional ned = newPeople.stream().filter(r -> r.getValueString("firstName").equals("Ned")).findFirst(); + assertThat(ned).isPresent(); + assertNull(ned.get().getValue("id"), "the record that got inserted should not have its primary key set"); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // but note, homer (who was updated) would have had its primary key set too, as part of the internal processing that does the update. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Optional homer = newPeople.stream().filter(r -> r.getValueString("firstName").equals("Homer")).findFirst(); + assertThat(homer).isPresent(); + assertNotNull(homer.get().getValue("id"), "the record that got updated should have its primary key set"); + ////////////////////////////// // assert homer was updated // ////////////////////////////// @@ -136,12 +155,18 @@ class ReplaceActionTest extends BaseTest replaceInput.setOmitDmlAudit(true); replaceInput.setRecords(newPeople); replaceInput.setFilter(null); + replaceInput.setSetPrimaryKeyInInsertedRecords(true); ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput); assertEquals(2, replaceOutput.getInsertOutput().getRecords().size()); assertEquals(0, replaceOutput.getUpdateOutput().getRecords().size()); assertEquals(2, replaceOutput.getDeleteOutput().getDeletedRecordCount()); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // due to true for SetPrimaryKeyInInsertedRecords, make sure primary keys ARE on all the records that got inserted // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertTrue(newPeople.stream().allMatch(r -> r.getValue("id") != null), "All inserted records should have their primary key"); + /////////////////////////////////////// // assert homer & marge were deleted // /////////////////////////////////////// From 889697f86f2859ee1a67f5503d45ba9f151fd35d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 10 May 2024 15:16:59 -0500 Subject: [PATCH 09/13] CE-1180 Better built-in support for processes with dynamic flows - that is: - in a backendStepOutput, you can set overrideLastStepName and call updateStepList - those values then flow through RunProcessAction to the runProcessOutput, then out through the javalin to the frontend. --- docs/metaData/Processes.adoc | 117 +++++++++- .../actions/processes/RunProcessAction.java | 34 ++- .../processes/RunBackendStepOutput.java | 128 +++++++++++ .../actions/processes/RunProcessOutput.java | 35 +++ .../RunProcessUpdateStepListTest.java | 211 ++++++++++++++++++ .../javalin/QJavalinProcessHandler.java | 7 + 6 files changed, 522 insertions(+), 10 deletions(-) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessUpdateStepListTest.java diff --git a/docs/metaData/Processes.adoc b/docs/metaData/Processes.adoc index 253f6318..2232b3f7 100644 --- a/docs/metaData/Processes.adoc +++ b/docs/metaData/Processes.adoc @@ -16,6 +16,7 @@ Processes are defined in a QQQ Instance in a `*QProcessMetaData*` object. In addition to directly building a `QProcessMetaData` object setting its properties directly, there are a few common process patterns that provide *Builder* objects for ease-of-use. See StreamedETLWithFrontendProcess below for a common example +[#_QProcessMetaData_Properties] *QProcessMetaData Properties:* * `name` - *String, Required* - Unique name for the process within the QQQ Instance. @@ -30,12 +31,13 @@ See below for details. * `permissionRules` - *QPermissionRules object* - define the permission/access rules for the process. See {link-permissionRules} for details. * `steps` and `stepList` - *Map of String → <>* and *List of QStepMetaData* - Defines the <> and <> that makes up the process. -** `stepList` is the list of steps in the order that they will by default be executed. -** `steps` is a map, including all steps from `stepList`, but which may also include steps which can used by the process if its backend steps make the decision to do so, at run-time. +** `stepList` is the list of steps in the order that they will be executed +(that is to say - this is the _default_ order of execution - but it can be customized - see <<_custom_process_flow>> for details). +** `steps` is a map, including all steps from `stepList`, but which may also include steps which can used by the process if its backend steps make the decision to do so, at run-time (e.g., using <<_custom_process_flow>>). ** A process's steps are normally defined in one of two was: *** 1) by a single call to `.withStepList(List)`, which internally adds each step into the `steps` map. *** 2) by multiple calls to `.addStep(QStepMetaData)`, which adds a step to both the `stepList` and `steps` map. -** If a process also needs optional steps, they should be added by a call to `.addOptionalStep(QStepMetaData)`, which only places them in the `steps` map. +** If a process also needs optional steps (for a <<_custom_process_flow>>), they should be added by a call to `.addOptionalStep(QStepMetaData)`, which only places them in the `steps` map. * `schedule` - *<>* - set up the process to run automatically on the specified schedule. See below for details. * `minInputRecords` - *Integer* - #not used...# @@ -214,3 +216,112 @@ But for some cases, doing page-level transactions can reduce long-transactions a * `withFields(List fieldList)` - Adds additional input fields to the preview step of the process. * `withBasepullConfiguration(BasepullConfiguration basepullConfiguration)` - Add a <> to the process. * `withSchedule(QScheduleMetaData schedule)` - Add a <> to the process. + +[#_custom_process_flow] +==== Custom Process Flow +As referenced in the definition of the <<_QProcessMetaData_Properties,QProcessMetaData Properties>>, by default, a process +will execute each of its steps in-order, as defined in the `stepList` property. +However, a Backend Step can customize this flow #todo - write more clearly here... + +There are generally 2 method to call (in a `BackendStep`) to do a dynamic flow: + +* `RunBackendStepOutput.setOverrideLastStepName(String stepName)` +** QQQ's `RunProcessAction` keeps track of which step it "last" ran, e.g., to tell it which one to run next. +However, if a step sets the `OverrideLastStepName` property in its output object, +then the step named in that property becomes the effective "last" step, +thus determining which step comes next. + +* `RunBackendStepOutput.updateStepList(List stepNameList)` +** Calling this method changes the process's runtime definition of steps to be executed. +Thus allowing a completely custom flow. +It should be noted, that the "last" step name (as tracked by QQQ within `RunProcessAction`) +does need to be found in the new `stepNameList` - otherwise, the framework will not know where you were, +for figuring out where to go next. + +[source,java] +.Example of a defining process that can use a flexible flow: +---- +// for a case like this, it would be recommended to define all step names in constants: +public final static String STEP_START = "start"; +public final static String STEP_A = "a"; +public final static String STEP_B = "b"; +public final static String STEP_C = "c"; +public final static String STEP_1 = "1"; +public final static String STEP_2 = "2"; +public final static String STEP_3 = "3"; +public final static String STEP_END = "end"; + +// also, to define the possible flows (lists of steps) in constants as well: +public final static List LETTERS_STEP_LIST = List.of( + STEP_START, STEP_A, STEP_B, STEP_C, STEP_END); + +public final static List NUMBERS_STEP_LIST = List.of( + STEP_START, STEP_1, STEP_2, STEP_3, STEP_END); + +// when we define the process's meta-data, we only give a "skeleton" stepList - +// we must at least have our starting step, and we may want at least one frontend step +// for the UI to show some placeholder(s): +QProcessMetaData process = new QProcessMetaData() + .withName(PROCESS_NAME) + .withStepList(List.of( + new QBackendStepMetaData().withName(STEP_START) + .withCode(new QCodeReference(/*...*/)), + new QFrontendStepMetaData() + .withName(STEP_END) + )); + +// the additional steps get added via `addOptionalStep`, which only puts them in +// the process's stepMap, not its stepList! +process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_A)); +process.addOptionalStep(new QBackendStepMetaData().withName(STEP_B) + .withCode(new QCodeReference(/*...*/))); +process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_C)); + +process.addOptionalStep(new QBackendStepMetaData().withName(STEP_1) + .withCode(new QCodeReference(/*...*/))); +process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_2)); +process.addOptionalStep(new QBackendStepMetaData().withName(STEP_3) + .withCode(new QCodeReference(/*...*/))); + +---- + +[source,java] +.Example of a process backend step adjusting the process's runtime flow: +---- +/*************************************************************************** +** look at the value named "which". if it's "letters", then make the process +** go through the stepList consisting of letters; else, update the step list +** to be the "numbers" steps. +** +** Also - if the "skipSomeSteps" value is give as true, then set the +** overrideLastStepName to skip again (in the letters case, skip past A, B +** and C; in the numbers case, skip past 1 and 2). +***************************************************************************/ +public static class StartStep implements BackendStep +{ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + Boolean skipSomeSteps = runBackendStepInput.getValueBoolean("skipSomeSteps"); + + if(runBackendStepInput.getValueString("which").equals("letters")) + { + runBackendStepOutput.updateStepList(LETTERS_STEP_LIST); + if(BooleanUtils.isTrue(skipSomeSteps)) + { + runBackendStepOutput.setOverrideLastStepName(STEP_C); + } + } + else + { + runBackendStepOutput.updateStepList(NUMBERS_STEP_LIST); + if(BooleanUtils.isTrue(skipSomeSteps)) + { + runBackendStepOutput.setOverrideLastStepName(STEP_2); + } + } + } +} +---- + + diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index 658bd65f..d2b82ee2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -190,7 +190,25 @@ public class RunProcessAction // Run backend steps // /////////////////////// LOG.debug("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]"); - runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState); + RunBackendStepOutput runBackendStepOutput = runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState); + + ///////////////////////////////////////////////////////////////////////////////////////// + // if the step returned an override lastStepName, use that to determine how we proceed // + ///////////////////////////////////////////////////////////////////////////////////////// + if(runBackendStepOutput.getOverrideLastStepName() != null) + { + LOG.debug("Process step [" + lastStepName + "] returned an overrideLastStepName [" + runBackendStepOutput.getOverrideLastStepName() + "]!"); + lastStepName = runBackendStepOutput.getOverrideLastStepName(); + } + + ///////////////////////////////////////////////////////////////////////////////////////////// + // similarly, if the step produced an updatedFrontendStepList, propagate that data outward // + ///////////////////////////////////////////////////////////////////////////////////////////// + if(runBackendStepOutput.getUpdatedFrontendStepList() != null) + { + LOG.debug("Process step [" + lastStepName + "] generated an updatedFrontendStepList [" + runBackendStepOutput.getUpdatedFrontendStepList().stream().map(s -> s.getName()).toList() + "]!"); + runProcessOutput.setUpdatedFrontendStepList(runBackendStepOutput.getUpdatedFrontendStepList()); + } } else { @@ -339,7 +357,7 @@ public class RunProcessAction /******************************************************************************* ** Run a single backend step. *******************************************************************************/ - private void runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception + private RunBackendStepOutput runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception { RunBackendStepInput runBackendStepInput = new RunBackendStepInput(processState); runBackendStepInput.setProcessName(process.getName()); @@ -368,14 +386,16 @@ public class RunProcessAction runBackendStepInput.setBasepullLastRunTime((Instant) runProcessInput.getValues().get(BASEPULL_LAST_RUNTIME_KEY)); } - RunBackendStepOutput lastFunctionResult = new RunBackendStepAction().execute(runBackendStepInput); - storeState(stateKey, lastFunctionResult.getProcessState()); + RunBackendStepOutput runBackendStepOutput = new RunBackendStepAction().execute(runBackendStepInput); + storeState(stateKey, runBackendStepOutput.getProcessState()); - if(lastFunctionResult.getException() != null) + if(runBackendStepOutput.getException() != null) { - runProcessOutput.setException(lastFunctionResult.getException()); - throw (lastFunctionResult.getException()); + runProcessOutput.setException(runBackendStepOutput.getException()); + throw (runBackendStepOutput.getException()); } + + return (runBackendStepOutput); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java index 9a915141..4cb6aa3b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java @@ -27,10 +27,14 @@ import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput; import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -40,9 +44,14 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public class RunBackendStepOutput extends AbstractActionOutput implements Serializable { + private String processName; + private ProcessState processState; private Exception exception; // todo - make optional + private String overrideLastStepName; + private List updatedFrontendStepList = null; + private List auditInputList = new ArrayList<>(); @@ -78,6 +87,7 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial public void seedFromRequest(RunBackendStepInput runBackendStepInput) { this.processState = runBackendStepInput.getProcessState(); + this.processName = runBackendStepInput.getProcessName(); } @@ -312,4 +322,122 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial auditInput.addAuditSingleInput(auditSingleInput); } + + + /******************************************************************************* + ** Getter for overrideLastStepName + *******************************************************************************/ + public String getOverrideLastStepName() + { + return (this.overrideLastStepName); + } + + + + /******************************************************************************* + ** Setter for overrideLastStepName + *******************************************************************************/ + public void setOverrideLastStepName(String overrideLastStepName) + { + this.overrideLastStepName = overrideLastStepName; + } + + + + /******************************************************************************* + ** Fluent setter for overrideLastStepName + *******************************************************************************/ + public RunBackendStepOutput withOverrideLastStepName(String overrideLastStepName) + { + this.overrideLastStepName = overrideLastStepName; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void updateStepList(List stepList) + { + getProcessState().setStepList(stepList); + + if(processName == null) + { + throw (new QRuntimeException("ProcessName was not set in this object, therefore updateStepList cannot complete successfully. Try to manually call setProcessName as a work around.")); + } + + QProcessMetaData processMetaData = QContext.getQInstance().getProcess(processName); + + ArrayList updatedFrontendStepList = new ArrayList<>(stepList.stream() + .map(name -> processMetaData.getStep(name)) + .filter(step -> step instanceof QFrontendStepMetaData) + .map(step -> (QFrontendStepMetaData) step) + .toList()); + + setUpdatedFrontendStepList(updatedFrontendStepList); + } + + + + /******************************************************************************* + ** Getter for processName + *******************************************************************************/ + public String getProcessName() + { + return (this.processName); + } + + + + /******************************************************************************* + ** Setter for processName + *******************************************************************************/ + public void setProcessName(String processName) + { + this.processName = processName; + } + + + + /******************************************************************************* + ** Fluent setter for processName + *******************************************************************************/ + public RunBackendStepOutput withProcessName(String processName) + { + this.processName = processName; + return (this); + } + + + + /******************************************************************************* + ** Getter for updatedFrontendStepList + *******************************************************************************/ + public List getUpdatedFrontendStepList() + { + return (this.updatedFrontendStepList); + } + + + + /******************************************************************************* + ** Setter for updatedFrontendStepList + *******************************************************************************/ + public void setUpdatedFrontendStepList(List updatedFrontendStepList) + { + this.updatedFrontendStepList = updatedFrontendStepList; + } + + + + /******************************************************************************* + ** Fluent setter for updatedFrontendStepList + *******************************************************************************/ + public RunBackendStepOutput withUpdatedFrontendStepList(List updatedFrontendStepList) + { + this.updatedFrontendStepList = updatedFrontendStepList; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessOutput.java index 466e02c4..7e05a660 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessOutput.java @@ -32,6 +32,7 @@ import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -45,6 +46,8 @@ public class RunProcessOutput extends AbstractActionOutput implements Serializab private String processUUID; private Optional exception = Optional.empty(); + private List updatedFrontendStepList = null; + /******************************************************************************* @@ -327,4 +330,36 @@ public class RunProcessOutput extends AbstractActionOutput implements Serializab { return exception; } + + + + /******************************************************************************* + ** Getter for updatedFrontendStepList + *******************************************************************************/ + public List getUpdatedFrontendStepList() + { + return (this.updatedFrontendStepList); + } + + + + /******************************************************************************* + ** Setter for updatedFrontendStepList + *******************************************************************************/ + public void setUpdatedFrontendStepList(List updatedFrontendStepList) + { + this.updatedFrontendStepList = updatedFrontendStepList; + } + + + + /******************************************************************************* + ** Fluent setter for updatedFrontendStepList + *******************************************************************************/ + public RunProcessOutput withUpdatedFrontendStepList(List updatedFrontendStepList) + { + this.updatedFrontendStepList = updatedFrontendStepList; + return (this); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessUpdateStepListTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessUpdateStepListTest.java new file mode 100644 index 00000000..7ae1ab99 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessUpdateStepListTest.java @@ -0,0 +1,211 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.processes; + + +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessTest.NoopBackendStep; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import org.apache.commons.lang.BooleanUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class RunProcessUpdateStepListTest extends BaseTest +{ + private static final String PROCESS_NAME = RunProcessUpdateStepListTest.class.getSimpleName(); + + private final static String STEP_START = "start"; + private final static String STEP_A = "a"; + private final static String STEP_B = "b"; + private final static String STEP_C = "c"; + private final static String STEP_1 = "1"; + private final static String STEP_2 = "2"; + private final static String STEP_3 = "3"; + private final static String STEP_END = "end"; + + private final static List LETTERS_STEP_LIST = List.of( + STEP_START, + STEP_A, + STEP_B, + STEP_C, + STEP_END + ); + + private final static List NUMBERS_STEP_LIST = List.of( + STEP_START, + STEP_1, + STEP_2, + STEP_3, + STEP_END + ); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGoingLettersPath() throws QException + { + QContext.getQInstance().addProcess(defineProcess()); + + //////////////////////////////////////////////////////////// + // start the process, telling it to go the "letters" path // + //////////////////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(PROCESS_NAME); + runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + runProcessInput.setValues(MapBuilder.of("which", "letters")); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // assert that we got back the next-step name of A, and the updated list of frontend steps (A, C) // + //////////////////////////////////////////////////////////////////////////////////////////////////// + Optional nextStepName = runProcessOutput.getProcessState().getNextStepName(); + assertTrue(nextStepName.isPresent()); + assertEquals(STEP_A, nextStepName.get()); + assertEquals(List.of(STEP_A, STEP_C, STEP_END), runProcessOutput.getUpdatedFrontendStepList().stream().map(s -> s.getName()).toList()); + + ///////////////////////////////////////////////// + // resume the process after that frontend step // + ///////////////////////////////////////////////// + runProcessInput.setProcessUUID(runProcessOutput.getProcessUUID()); + runProcessInput.setStartAfterStep(nextStepName.get()); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + + /////////////////////////////////////////////////////////////////////////////////////// + // assert we got back C as the next-step now, and no updated frontend list this time // + /////////////////////////////////////////////////////////////////////////////////////// + nextStepName = runProcessOutput.getProcessState().getNextStepName(); + assertTrue(nextStepName.isPresent()); + assertEquals(STEP_C, nextStepName.get()); + assertNull(runProcessOutput.getUpdatedFrontendStepList()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGoingNumbersPathAndSkippingAhead() throws QException + { + QContext.getQInstance().addProcess(defineProcess()); + + //////////////////////////////////////////////////////////////////////////////////// + // start the process, telling it to go the "numbers" path, and to skip ahead some // + //////////////////////////////////////////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(PROCESS_NAME); + runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + runProcessInput.setValues(MapBuilder.of("which", "numbers", "skipSomeSteps", true)); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // assert that we got back the next-step name of 2, and the updated list of frontend steps (1, 3) // + //////////////////////////////////////////////////////////////////////////////////////////////////// + Optional nextStepName = runProcessOutput.getProcessState().getNextStepName(); + assertTrue(nextStepName.isPresent()); + assertEquals(STEP_END, nextStepName.get()); + assertEquals(List.of(STEP_2, STEP_END), runProcessOutput.getUpdatedFrontendStepList().stream().map(s -> s.getName()).toList()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QProcessMetaData defineProcess() + { + QProcessMetaData process = new QProcessMetaData() + .withName(PROCESS_NAME) + .withStepList(List.of( + new QBackendStepMetaData() + .withName(STEP_START) + .withCode(new QCodeReference(StartStep.class)), + new QFrontendStepMetaData() + .withName(STEP_END) + )); + + process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_A)); + process.addOptionalStep(new QBackendStepMetaData().withName(STEP_B).withCode(new QCodeReference(NoopBackendStep.class))); + process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_C)); + + process.addOptionalStep(new QBackendStepMetaData().withName(STEP_1).withCode(new QCodeReference(NoopBackendStep.class))); + process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_2)); + process.addOptionalStep(new QBackendStepMetaData().withName(STEP_3).withCode(new QCodeReference(NoopBackendStep.class))); + + return (process); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class StartStep implements BackendStep + { + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + boolean skipSomeSteps = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("skipSomeSteps")); + + if(runBackendStepInput.getValueString("which").equals("letters")) + { + runBackendStepOutput.updateStepList(LETTERS_STEP_LIST); + if(skipSomeSteps) + { + runBackendStepOutput.setOverrideLastStepName(STEP_C); + } + } + else + { + runBackendStepOutput.updateStepList(NUMBERS_STEP_LIST); + if(skipSomeSteps) + { + runBackendStepOutput.setOverrideLastStepName(STEP_2); + } + } + } + } +} diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java index d4238f64..957b5a21 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java @@ -72,6 +72,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -448,6 +449,12 @@ public class QJavalinProcessHandler } resultForCaller.put("values", runProcessOutput.getValues()); runProcessOutput.getProcessState().getNextStepName().ifPresent(nextStep -> resultForCaller.put("nextStep", nextStep)); + + List updatedFrontendStepList = runProcessOutput.getUpdatedFrontendStepList(); + if(updatedFrontendStepList != null) + { + resultForCaller.put("updatedFrontendStepList", updatedFrontendStepList); + } } From baac007c09debb3766cbd07c3c5dc5f2818615cd Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 13 May 2024 08:41:38 -0500 Subject: [PATCH 10/13] CE-1180 Add QBadHttpResponseStatusException, to make easier for implementations to see what status code there was in a failure --- .../module/api/actions/BaseAPIActionUtil.java | 3 +- .../QBadHttpResponseStatusException.java | 78 +++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/QBadHttpResponseStatusException.java diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 5be4635d..700664c9 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -69,6 +69,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.module.api.exceptions.OAuthCredentialsException; import com.kingsrook.qqq.backend.module.api.exceptions.OAuthExpiredTokenException; +import com.kingsrook.qqq.backend.module.api.exceptions.QBadHttpResponseStatusException; import com.kingsrook.qqq.backend.module.api.exceptions.RateLimitException; import com.kingsrook.qqq.backend.module.api.exceptions.RetryableServerErrorException; import com.kingsrook.qqq.backend.module.api.model.AuthorizationType; @@ -593,7 +594,7 @@ public class BaseAPIActionUtil } String warningMessage = "HTTP " + request.getMethod() + " for table [" + table.getName() + "] failed with status " + statusCode + ": " + resultString; - throw (new QException(warningMessage)); + throw (new QBadHttpResponseStatusException(warningMessage, statusCode)); } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/QBadHttpResponseStatusException.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/QBadHttpResponseStatusException.java new file mode 100644 index 00000000..35ddd0cd --- /dev/null +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/QBadHttpResponseStatusException.java @@ -0,0 +1,78 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.api.exceptions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; + + +/******************************************************************************* + ** Exception thrown when an API HTTP request failed due to a bad status code. + ** This exception includes the status code as a field + *******************************************************************************/ +public class QBadHttpResponseStatusException extends QException +{ + private int statusCode; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QBadHttpResponseStatusException(String message, int statusCode) + { + super(message); + this.statusCode = statusCode; + } + + + + /******************************************************************************* + ** Getter for statusCode + *******************************************************************************/ + public int getStatusCode() + { + return (this.statusCode); + } + + + + /******************************************************************************* + ** Setter for statusCode + *******************************************************************************/ + public void setStatusCode(int statusCode) + { + this.statusCode = statusCode; + } + + + + /******************************************************************************* + ** Fluent setter for statusCode + *******************************************************************************/ + public QBadHttpResponseStatusException withStatusCode(int statusCode) + { + this.statusCode = statusCode; + return (this); + } + +} From a06db0b7a8c75063aea13e96d13357bbeb9f2423 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 13 May 2024 08:42:09 -0500 Subject: [PATCH 11/13] CE-1180 Add method getAllSteps --- .../model/metadata/processes/QProcessMetaData.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index fae291e8..44c9259b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -736,4 +736,14 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi } + + /******************************************************************************* + ** Getter for the full map of all steps (not the step list!) + ** + *******************************************************************************/ + public Map getAllSteps() + { + return steps; + } + } From e7735619c18f89fb9c65bf9ab8fd6a747aa34620 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 13 May 2024 08:42:29 -0500 Subject: [PATCH 12/13] CE-1180 Add clone --- .../actions/tables/query/QueryInput.java | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java index 8b88008e..a0e71e19 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; @@ -37,7 +38,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterf ** Input data for the Query action ** *******************************************************************************/ -public class QueryInput extends AbstractTableActionInput implements QueryOrGetInputInterface +public class QueryInput extends AbstractTableActionInput implements QueryOrGetInputInterface, Cloneable { private QBackendTransaction transaction; private QQueryFilter filter; @@ -109,6 +110,40 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryInput clone() throws CloneNotSupportedException + { + QueryInput clone = (QueryInput) super.clone(); + + if(fieldsToTranslatePossibleValues != null) + { + clone.fieldsToTranslatePossibleValues = new HashSet<>(fieldsToTranslatePossibleValues); + } + + if(queryJoins != null) + { + clone.queryJoins = new ArrayList<>(queryJoins); + } + + if(clone.associationNamesToInclude != null) + { + clone.associationNamesToInclude = new HashSet<>(associationNamesToInclude); + } + + if(queryHints != null) + { + clone.queryHints = EnumSet.noneOf(QueryHint.class); + clone.queryHints.addAll(queryHints); + } + + return (clone); + } + + + /******************************************************************************* ** Getter for filter ** From 7a1b99bab326c6c9757a8979d423548025826702 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 14 May 2024 08:31:40 -0500 Subject: [PATCH 13/13] CE-1180 Add full QHttpResponse to BadResponse exception --- .../module/api/actions/BaseAPIActionUtil.java | 2 +- .../QBadHttpResponseStatusException.java | 44 +++++++++++++++++-- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 700664c9..57cef42d 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -594,7 +594,7 @@ public class BaseAPIActionUtil } String warningMessage = "HTTP " + request.getMethod() + " for table [" + table.getName() + "] failed with status " + statusCode + ": " + resultString; - throw (new QBadHttpResponseStatusException(warningMessage, statusCode)); + throw (new QBadHttpResponseStatusException(warningMessage, response)); } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/QBadHttpResponseStatusException.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/QBadHttpResponseStatusException.java index 35ddd0cd..8587ba29 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/QBadHttpResponseStatusException.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/QBadHttpResponseStatusException.java @@ -23,25 +23,30 @@ package com.kingsrook.qqq.backend.module.api.exceptions; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.module.api.actions.QHttpResponse; /******************************************************************************* ** Exception thrown when an API HTTP request failed due to a bad status code. - ** This exception includes the status code as a field + ** This exception includes the status code as a field, as well as the full + ** response object. *******************************************************************************/ public class QBadHttpResponseStatusException extends QException { - private int statusCode; + private int statusCode; + private QHttpResponse response; /******************************************************************************* ** *******************************************************************************/ - public QBadHttpResponseStatusException(String message, int statusCode) + public QBadHttpResponseStatusException(String message, QHttpResponse response) { super(message); - this.statusCode = statusCode; + + this.statusCode = response.getStatusCode(); + this.response = response; } @@ -75,4 +80,35 @@ public class QBadHttpResponseStatusException extends QException return (this); } + + + /******************************************************************************* + ** Getter for response + *******************************************************************************/ + public QHttpResponse getResponse() + { + return (this.response); + } + + + + /******************************************************************************* + ** Setter for response + *******************************************************************************/ + public void setResponse(QHttpResponse response) + { + this.response = response; + } + + + + /******************************************************************************* + ** Fluent setter for response + *******************************************************************************/ + public QBadHttpResponseStatusException withResponse(QHttpResponse response) + { + this.response = response; + return (this); + } + }