From fa80daa778cc4ad38b5160ad9c0760124067d867 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 3 Jul 2025 07:57:37 -0500 Subject: [PATCH 1/6] Add method addAllIfNotNull --- .../backend/core/utils/CollectionUtils.java | 23 +++++++++++++ .../core/utils/CollectionUtilsTest.java | 34 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java index 90e2756d..17ac02fd 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Function; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.ObjectMapper; @@ -712,4 +713,26 @@ public class CollectionUtils c.add(element); } } + + + + /*************************************************************************** + * add all objects in a source collection to a destination collection, in a + * null-safe manner with regard to the source. + * + * @param destination collection to put objects into. May NOT be null. + * if it's immutable, and source is not null, that will + * fail (as you'd expect) too. + * @param source collection to get objects from. if null, is ignored. + * @throws NullPointerException if destination is null. + ***************************************************************************/ + public static void addAllIfNotNull(Collection destination, Collection source) + { + Objects.requireNonNull(destination, "destination may not be null"); + + if(source != null) + { + destination.addAll(source); + } + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/CollectionUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/CollectionUtilsTest.java index a243c775..7ccc9780 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/CollectionUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/CollectionUtilsTest.java @@ -33,11 +33,14 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; +import java.util.function.BiFunction; import java.util.function.Function; import com.google.gson.reflect.TypeToken; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -639,4 +642,35 @@ class CollectionUtilsTest extends BaseTest assertEquals(Set.of("", "1"), s); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testAddAllIfNotNull() + { + BiFunction, Collection, Collection> doAddAllIfNotNull = (Collection destination, Collection source) -> + { + CollectionUtils.addAllIfNotNull(destination, source); + return (destination); + }; + + assertThatThrownBy(() -> doAddAllIfNotNull.apply(null, null)).hasMessage("destination may not be null"); + assertThatThrownBy(() -> doAddAllIfNotNull.apply(null, Collections.emptyList())).hasMessage("destination may not be null"); + assertThatThrownBy(() -> doAddAllIfNotNull.apply(null, List.of(1))).hasMessage("destination may not be null"); + + assertEquals(List.of(), doAddAllIfNotNull.apply(new ArrayList<>(), null)); + assertEquals(List.of(), doAddAllIfNotNull.apply(new ArrayList<>(), Collections.emptyList())); + assertEquals(List.of(1), doAddAllIfNotNull.apply(new ArrayList<>(), List.of(1))); + + assertEquals(List.of(1), doAddAllIfNotNull.apply(ListBuilder.of(1), null)); + assertEquals(List.of(1, 2), doAddAllIfNotNull.apply(ListBuilder.of(1), ListBuilder.of(2))); + assertEquals(List.of(1, 2, 3), doAddAllIfNotNull.apply(ListBuilder.of(1), ListBuilder.of(2, 3))); + + assertEquals(Set.of(1), doAddAllIfNotNull.apply(new HashSet<>(List.of(1)), null)); + assertEquals(Set.of(1, 2), doAddAllIfNotNull.apply(new HashSet<>(List.of(1)), List.of(2))); + assertEquals(Set.of(1, 2, 3), doAddAllIfNotNull.apply(new HashSet<>(List.of(1)), List.of(2, 3))); + } + } From f97a3d50975ceb87ff2c701967b0e44238189890 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 3 Jul 2025 07:59:40 -0500 Subject: [PATCH 2/6] Pass transaction through from insert/update/delete actions through DMLAuditAction into AuditAction --- .../core/actions/audits/AuditAction.java | 2 + .../core/actions/audits/DMLAuditAction.java | 1 + .../core/actions/tables/DeleteAction.java | 1 + .../core/actions/tables/InsertAction.java | 1 + .../core/actions/tables/UpdateAction.java | 1 + .../core/model/actions/audits/AuditInput.java | 41 ++++++++++++++++++ .../model/actions/audits/DMLAuditInput.java | 42 +++++++++++++++++++ 7 files changed, 89 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java index 031ca851..27e04ff2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java @@ -291,6 +291,7 @@ public class AuditAction extends AbstractQActionFunction dmlAuditInput.setRecordList(l)); new DMLAuditAction().execute(dmlAuditInput); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index e56da2b7..01939989 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -170,6 +170,7 @@ public class InsertAction extends AbstractQActionFunction auditSingleInputList = new ArrayList<>(); + private QBackendTransaction transaction; + /******************************************************************************* @@ -92,4 +95,42 @@ public class AuditInput extends AbstractActionInput implements Serializable return (this); } + + /******************************************************************************* + * Getter for transaction + * @see #withTransaction(QBackendTransaction) + *******************************************************************************/ + public QBackendTransaction getTransaction() + { + return (this.transaction); + } + + + + /******************************************************************************* + * Setter for transaction + * @see #withTransaction(QBackendTransaction) + *******************************************************************************/ + public void setTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + } + + + + /******************************************************************************* + * Fluent setter for transaction + * + * @param transaction + * transaction upon which the audits will be inserted. + * + * @return this + *******************************************************************************/ + public AuditInput withTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/DMLAuditInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/DMLAuditInput.java index 654726c5..79be0213 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/DMLAuditInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/DMLAuditInput.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.audits; import java.io.Serializable; import java.util.List; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -38,6 +39,8 @@ public class DMLAuditInput extends AbstractActionInput implements Serializable private List oldRecordList; private AbstractTableActionInput tableActionInput; + private QBackendTransaction transaction; + private String auditContext = null; @@ -164,4 +167,43 @@ public class DMLAuditInput extends AbstractActionInput implements Serializable return (this); } + + /******************************************************************************* + * Getter for transaction + * @see #withTransaction(QBackendTransaction) + *******************************************************************************/ + public QBackendTransaction getTransaction() + { + return (this.transaction); + } + + + + /******************************************************************************* + * Setter for transaction + * @see #withTransaction(QBackendTransaction) + *******************************************************************************/ + public void setTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + } + + + + /******************************************************************************* + * Fluent setter for transaction + * + * @param transaction + * transaction that will be used for inserting the audits, where (presumably) + * the DML against the record occurred as well + * + * @return this + *******************************************************************************/ + public DMLAuditInput withTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + return (this); + } + + } From 946e7d418b56679e20685acaeea39f80f5db6a8a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 3 Jul 2025 08:04:07 -0500 Subject: [PATCH 3/6] Add method get(Class,String) --- .../metadata/MetaDataProducerMultiOutput.java | 32 +++++++ .../MetaDataProducerMultiOutputTest.java | 84 +++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutputTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java index 2ec4dea8..f473aec8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java @@ -145,4 +145,36 @@ public class MetaDataProducerMultiOutput implements MetaDataProducerOutput, Sour setSourceQBitName(sourceQBitName); return this; } + + + + /*************************************************************************** + * get a typed and named meta-data object out of this output container. + * + * @param the type of the object to return, e.g., QTableMetaData + * @param outputClass the class for the type to return + * @param name the name of the object, e.g., a table or process name. + * @return the requested TopLevelMetaDataInterface object (in the requested + * type), or null if not found. + ***************************************************************************/ + public C get(Class outputClass, String name) + { + for(MetaDataProducerOutput content : CollectionUtils.nonNullList(contents)) + { + if(content instanceof MetaDataProducerMultiOutput multiOutput) + { + C c = multiOutput.get(outputClass, name); + if(c != null) + { + return (c); + } + } + else if(outputClass.isInstance(content) && name.equals(((TopLevelMetaDataInterface)content).getName())) + { + return (C) content; + } + } + + return null; + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutputTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutputTest.java new file mode 100644 index 00000000..53a6b888 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutputTest.java @@ -0,0 +1,84 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for MetaDataProducerMultiOutput + *******************************************************************************/ +class MetaDataProducerMultiOutputTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetEachAndGet() + { + ////////////////////// + // given this setup // + ////////////////////// + MetaDataProducerMultiOutput metaDataProducerMultiOutput = new MetaDataProducerMultiOutput(); + metaDataProducerMultiOutput.add(new QTableMetaData().withName("tableA")); + metaDataProducerMultiOutput.add(new QProcessMetaData().withName("processB")); + metaDataProducerMultiOutput.add(new QBackendMetaData().withName("backendC")); + metaDataProducerMultiOutput.add(new QTableMetaData().withName("tableD")); + + /////////////////////////// + // test calls to getEach // + /////////////////////////// + List tables = metaDataProducerMultiOutput.getEach(QTableMetaData.class); + assertEquals(2, tables.size()); + assertEquals("tableA", tables.get(0).getName()); + assertEquals("tableD", tables.get(1).getName()); + + List processes = metaDataProducerMultiOutput.getEach(QProcessMetaData.class); + assertEquals(1, processes.size()); + assertEquals("processB", processes.get(0).getName()); + + List backends = metaDataProducerMultiOutput.getEach(QBackendMetaData.class); + assertEquals(1, backends.size()); + assertEquals("backendC", backends.get(0).getName()); + + List queueProviders = metaDataProducerMultiOutput.getEach(QQueueProviderMetaData.class); + assertEquals(0, queueProviders.size()); + + ////////////////////////////////////////////// + // test some calls to get that takes a name // + ////////////////////////////////////////////// + assertEquals("tableA", metaDataProducerMultiOutput.get(QTableMetaData.class, "tableA").getName()); + assertNull(metaDataProducerMultiOutput.get(QProcessMetaData.class, "tableA")); + assertNull(metaDataProducerMultiOutput.get(QQueueMetaData.class, "queueQ")); + } + +} \ No newline at end of file From ff1cf813154c0ec773c38296b04fed5c6a124f3e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 3 Jul 2025 08:04:34 -0500 Subject: [PATCH 4/6] Switch from testing for QBitComponentMetaDataProducer to use QBitComponentMetaDataProducerInterface instead --- .../core/model/metadata/qbits/QBitMetaDataProducer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaDataProducer.java index c37f466b..e7b4091a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaDataProducer.java @@ -68,7 +68,7 @@ public interface QBitMetaDataProducer extends MetaDataProd /*************************************************************************** ** ***************************************************************************/ - default void postProduceActions(MetaDataProducerMultiOutput metaDataProducerMultiOutput, QInstance qinstance) + default void postProduceActions(MetaDataProducerMultiOutput metaDataProducerMultiOutput, QInstance qinstance) throws QException { ///////////////////// // noop by default // @@ -137,9 +137,9 @@ public interface QBitMetaDataProducer extends MetaDataProd //////////////////////////////////////////////////////////////////////////// // todo is this deprecated in favor of QBitProductionContext's stack... ? // //////////////////////////////////////////////////////////////////////////// - if(producer instanceof QBitComponentMetaDataProducer) + if(producer instanceof QBitComponentMetaDataProducerInterface) { - QBitComponentMetaDataProducer qBitComponentMetaDataProducer = (QBitComponentMetaDataProducer) producer; + QBitComponentMetaDataProducerInterface qBitComponentMetaDataProducer = (QBitComponentMetaDataProducerInterface) producer; qBitComponentMetaDataProducer.setQBitConfig(qBitConfig); } From 3183dd028feccd456ce25d9f414b3465553cdba3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 3 Jul 2025 08:05:57 -0500 Subject: [PATCH 5/6] Add protection against ConcurrentModificationException when processing QSupplementalInstanceMetaData - for the case where enriching one adds another --- .../core/instances/QInstanceEnricher.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 0817c585..9f2269b1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -31,7 +31,9 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -43,6 +45,7 @@ import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph; import com.kingsrook.qqq.backend.core.actions.permissions.BulkTableActionProcessPermissionChecker; import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -212,9 +215,35 @@ public class QInstanceEnricher ***************************************************************************/ private void enrichInstance() { - for(QSupplementalInstanceMetaData supplementalInstanceMetaData : qInstance.getSupplementalMetaData().values()) + //////////////////////////////////////////////////////////////////////////////////// + // enriching some objects may cause additional ones to be added to the qInstance! // + // this caused concurrent modification exceptions, when we just iterated. // + // we could make a copy of the map and just process that, but then we wouldn't // + // enrich any new objects that do get added, so, use this technique instead. // + //////////////////////////////////////////////////////////////////////////////////// + Set toEnrich = new LinkedHashSet<>(qInstance.getSupplementalMetaData().values()); + Set enriched = new HashSet<>(); + int count = 0; + while(!toEnrich.isEmpty()) { + Iterator iterator = toEnrich.iterator(); + QSupplementalInstanceMetaData supplementalInstanceMetaData = iterator.next(); + iterator.remove(); + supplementalInstanceMetaData.enrich(qInstance); + enriched.add(supplementalInstanceMetaData); + + for(QSupplementalInstanceMetaData possiblyNew : qInstance.getSupplementalMetaData().values()) + { + if(!toEnrich.contains(possiblyNew) && !enriched.contains(possiblyNew)) + { + if(count++ > 100) + { + throw (new QRuntimeException("Too many new QSupplementalInstanceMetaData objects were added while enriching others. This probably indicates a bug in enrichment code. Throwing to prevent infinite loop.")); + } + toEnrich.add(possiblyNew); + } + } } runPlugins(QInstance.class, qInstance, qInstance); From 4788faae7d6331fd085b82888e83143fffb09385 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 3 Jul 2025 08:06:13 -0500 Subject: [PATCH 6/6] Mark as Serializable --- .../qqq/backend/core/processes/utils/RecordLookupHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java index 899fd468..070cd8ca 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java @@ -50,7 +50,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; ** can pre-load entire tables or subsets of tables. ** *******************************************************************************/ -public class RecordLookupHelper +public class RecordLookupHelper implements Serializable { private Map> recordMaps;