From 9af1fed422c4ef17f00392aa764cd2af2e22fe4c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 6 Jul 2023 18:53:10 -0500 Subject: [PATCH 01/24] Initial backend work to support datetime query expressions from frontend --- .../actions/tables/query/QFilterCriteria.java | 3 + .../expressions/AbstractFilterExpression.java | 21 +++ .../query/expressions/NowWithOffset.java | 62 +++++- .../query/expressions/ThisOrLastPeriod.java | 178 ++++++++++++++++++ .../QFilterCriteriaDeserializer.java | 124 ++++++++++++ .../query/expressions/NowWithOffsetTest.java | 68 +++++++ .../QFilterCriteriaDeserializerTest.java | 93 +++++++++ 7 files changed, 542 insertions(+), 7 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/ThisOrLastPeriod.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializer.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializerTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java index e096343e..f43f7eb9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java @@ -26,8 +26,10 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.serialization.QFilterCriteriaDeserializer; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -36,6 +38,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; * A single criteria Component of a Query * *******************************************************************************/ +@JsonDeserialize(using = QFilterCriteriaDeserializer.class) public class QFilterCriteria implements Serializable, Cloneable { private static final QLogger LOG = QLogger.getLogger(QFilterCriteria.class); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/AbstractFilterExpression.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/AbstractFilterExpression.java index cc75d062..723fa818 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/AbstractFilterExpression.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/AbstractFilterExpression.java @@ -34,4 +34,25 @@ public abstract class AbstractFilterExpression ** *******************************************************************************/ public abstract T evaluate(); + + + + /******************************************************************************* + ** To help with serialization, define a "type" in all subclasses + *******************************************************************************/ + public String getType() + { + return (getClass().getSimpleName()); + } + + + + /******************************************************************************* + ** noop - but here so serialization won't be upset about there being a type + ** in a json object. + *******************************************************************************/ + public void setType(String type) + { + + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffset.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffset.java index 9400313b..bf36d971 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffset.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffset.java @@ -23,6 +23,10 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions; import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; import java.util.concurrent.TimeUnit; @@ -31,9 +35,9 @@ import java.util.concurrent.TimeUnit; *******************************************************************************/ public class NowWithOffset extends AbstractFilterExpression { - private final Operator operator; - private final int amount; - private final TimeUnit timeUnit; + private Operator operator; + private int amount; + private ChronoUnit timeUnit; @@ -46,7 +50,17 @@ public class NowWithOffset extends AbstractFilterExpression ** Constructor ** *******************************************************************************/ - private NowWithOffset(Operator operator, int amount, TimeUnit timeUnit) + public NowWithOffset() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + private NowWithOffset(Operator operator, int amount, ChronoUnit timeUnit) { this.operator = operator; this.amount = amount; @@ -59,7 +73,19 @@ public class NowWithOffset extends AbstractFilterExpression ** Factory ** *******************************************************************************/ + @Deprecated public static NowWithOffset minus(int amount, TimeUnit timeUnit) + { + return (minus(amount, timeUnit.toChronoUnit())); + } + + + + /******************************************************************************* + ** Factory + ** + *******************************************************************************/ + public static NowWithOffset minus(int amount, ChronoUnit timeUnit) { return (new NowWithOffset(Operator.MINUS, amount, timeUnit)); } @@ -70,7 +96,19 @@ public class NowWithOffset extends AbstractFilterExpression ** Factory ** *******************************************************************************/ + @Deprecated public static NowWithOffset plus(int amount, TimeUnit timeUnit) + { + return (plus(amount, timeUnit.toChronoUnit())); + } + + + + /******************************************************************************* + ** Factory + ** + *******************************************************************************/ + public static NowWithOffset plus(int amount, ChronoUnit timeUnit) { return (new NowWithOffset(Operator.PLUS, amount, timeUnit)); } @@ -83,14 +121,24 @@ public class NowWithOffset extends AbstractFilterExpression @Override public Instant evaluate() { + ///////////////////////////////////////////////////////////////////////////// + // Instant doesn't let us plus/minus WEEK, MONTH, or YEAR... // + // but LocalDateTime does. So, make a LDT in UTC, do the plus/minus, then // + // convert back to Instant @ UTC // + ///////////////////////////////////////////////////////////////////////////// + LocalDateTime now = LocalDateTime.now(ZoneId.of("UTC")); + + LocalDateTime then; if(operator.equals(Operator.PLUS)) { - return (Instant.now().plus(amount, timeUnit.toChronoUnit())); + then = now.plus(amount, timeUnit); } else { - return (Instant.now().minus(amount, timeUnit.toChronoUnit())); + then = now.minus(amount, timeUnit); } + + return (then.toInstant(ZoneOffset.UTC)); } @@ -121,7 +169,7 @@ public class NowWithOffset extends AbstractFilterExpression ** Getter for timeUnit ** *******************************************************************************/ - public TimeUnit getTimeUnit() + public ChronoUnit getTimeUnit() { return timeUnit; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/ThisOrLastPeriod.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/ThisOrLastPeriod.java new file mode 100644 index 00000000..28e53c37 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/ThisOrLastPeriod.java @@ -0,0 +1,178 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions; + + +import java.time.DayOfWeek; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.temporal.ChronoField; +import java.time.temporal.ChronoUnit; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ThisOrLastPeriod extends AbstractFilterExpression +{ + private Operator operator; + private ChronoUnit timeUnit; + + + + public enum Operator + {THIS, LAST} + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ThisOrLastPeriod() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + private ThisOrLastPeriod(Operator operator, ChronoUnit timeUnit) + { + this.operator = operator; + this.timeUnit = timeUnit; + } + + + + /******************************************************************************* + ** Factory + ** + *******************************************************************************/ + public static ThisOrLastPeriod this_(ChronoUnit timeUnit) + { + return (new ThisOrLastPeriod(Operator.THIS, timeUnit)); + } + + + + /******************************************************************************* + ** Factory + ** + *******************************************************************************/ + public static ThisOrLastPeriod last(int amount, ChronoUnit timeUnit) + { + return (new ThisOrLastPeriod(Operator.LAST, timeUnit)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Instant evaluate() + { + ZoneId zoneId = ValueUtils.getSessionOrInstanceZoneId(); + + switch(timeUnit) + { + case HOURS -> + { + if(operator.equals(Operator.THIS)) + { + return Instant.now().truncatedTo(ChronoUnit.HOURS); + } + else + { + return Instant.now().minus(1, ChronoUnit.HOURS).truncatedTo(ChronoUnit.HOURS); + } + } + case DAYS -> + { + Instant startOfToday = ValueUtils.getStartOfTodayInZoneId(zoneId.getId()); + return operator.equals(Operator.THIS) ? startOfToday : startOfToday.minus(1, ChronoUnit.DAYS); + } + case WEEKS -> + { + Instant startOfToday = ValueUtils.getStartOfTodayInZoneId(zoneId.getId()); + LocalDateTime startOfThisWeekLDT = LocalDateTime.ofInstant(startOfToday, zoneId); + while(startOfThisWeekLDT.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue()) + { + //////////////////////////////////////// + // go backwards until sunday is found // + //////////////////////////////////////// + startOfThisWeekLDT = startOfThisWeekLDT.minus(1, ChronoUnit.DAYS); + } + Instant startOfThisWeek = startOfThisWeekLDT.toInstant(zoneId.getRules().getOffset(startOfThisWeekLDT)); + + return operator.equals(Operator.THIS) ? startOfThisWeek : startOfThisWeek.minus(7, ChronoUnit.DAYS); + } + case MONTHS -> + { + Instant startOfThisMonth = ValueUtils.getStartOfMonthInZoneId(zoneId.getId()); + LocalDateTime startOfThisMonthLDT = LocalDateTime.ofInstant(startOfThisMonth, ZoneId.of(zoneId.getId())); + LocalDateTime startOfLastMonthLDT = startOfThisMonthLDT.minus(1, ChronoUnit.MONTHS); + Instant startOfLastMonth = startOfLastMonthLDT.toInstant(ZoneId.of(zoneId.getId()).getRules().getOffset(Instant.now())); + + return operator.equals(Operator.THIS) ? startOfThisMonth : startOfLastMonth; + } + case YEARS -> + { + Instant startOfThisYear = ValueUtils.getStartOfYearInZoneId(zoneId.getId()); + LocalDateTime startOfThisYearLDT = LocalDateTime.ofInstant(startOfThisYear, zoneId); + LocalDateTime startOfLastYearLDT = startOfThisYearLDT.minus(1, ChronoUnit.YEARS); + Instant startOfLastYear = startOfLastYearLDT.toInstant(zoneId.getRules().getOffset(Instant.now())); + + return operator.equals(Operator.THIS) ? startOfThisYear : startOfLastYear; + } + default -> throw (new QRuntimeException("Unsupported timeUnit: " + timeUnit)); + } + } + + + + /******************************************************************************* + ** Getter for operator + ** + *******************************************************************************/ + public Operator getOperator() + { + return operator; + } + + + + /******************************************************************************* + ** Getter for timeUnit + ** + *******************************************************************************/ + public ChronoUnit getTimeUnit() + { + return timeUnit; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializer.java new file mode 100644 index 00000000..ae4dfd64 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializer.java @@ -0,0 +1,124 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.tables.query.serialization; + + +import java.io.IOException; +import java.util.List; +import com.fasterxml.jackson.core.JacksonException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression; + + +/******************************************************************************* + ** Custom jackson deserializer, to deal w/ abstract expression field + *******************************************************************************/ +public class QFilterCriteriaDeserializer extends StdDeserializer +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public QFilterCriteriaDeserializer() + { + this(null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QFilterCriteriaDeserializer(Class vc) + { + super(vc); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QFilterCriteria deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JacksonException + { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + ObjectMapper objectMapper = new ObjectMapper(); + + ///////////////////////////////// + // get values out of json node // + ///////////////////////////////// + List values = objectMapper.treeToValue(node.get("values"), List.class); + String fieldName = objectMapper.treeToValue(node.get("fieldName"), String.class); + QCriteriaOperator operator = objectMapper.treeToValue(node.get("operator"), QCriteriaOperator.class); + String otherFieldName = objectMapper.treeToValue(node.get("otherFieldName"), String.class); + + AbstractFilterExpression expression = null; + if(node.has("expression")) + { + JsonNode expressionNode = node.get("expression"); + String expressionType = objectMapper.treeToValue(expressionNode.get("type"), String.class); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // right now, we'll assume that all expression subclasses are in the same package as AbstractFilterExpression // + // so, we can just do a Class.forName on that name, and treeToValue like that. // + // if we ever had to, we could instead switch(expressionType), and do like so... // + // case "NowWithOffset" -> objectMapper.treeToValue(expressionNode, NowWithOffset.class); // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + try + { + String className = AbstractFilterExpression.class.getName().replace(AbstractFilterExpression.class.getSimpleName(), expressionType); + expression = (AbstractFilterExpression) objectMapper.treeToValue(expressionNode, Class.forName(className)); + } + catch(Exception e) + { + throw (new IOException("Error deserializing expression of type [" + expressionType + "] inside QFilterCriteria", e)); + } + } + + /////////////////////////////////// + // put fields into return object // + /////////////////////////////////// + QFilterCriteria criteria = new QFilterCriteria(); + criteria.setFieldName(fieldName); + criteria.setOperator(operator); + criteria.setValues(values); + criteria.setOtherFieldName(otherFieldName); + criteria.setExpression(expression); + + return (criteria); + + /* + int id = (Integer) (node.get("id")).numberValue(); + String itemName = node.get("itemName").asText(); + int userId = (Integer) (node.get("createdBy")).numberValue(); + + return null; + */ + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java new file mode 100644 index 00000000..8ef5737d --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java @@ -0,0 +1,68 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions; + + +import java.time.temporal.ChronoUnit; +import com.kingsrook.qqq.backend.core.BaseTest; +import org.assertj.core.data.Offset; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for NowWithOffset + *******************************************************************************/ +class NowWithOffsetTest extends BaseTest +{ + private static final long DAY_IN_MILLIS = 24 * 60 * 60 * 1000; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + long now = System.currentTimeMillis(); + + long oneWeekAgoMillis = NowWithOffset.minus(1, ChronoUnit.WEEKS).evaluate().toEpochMilli(); + assertThat(oneWeekAgoMillis).isCloseTo(now - (7 * DAY_IN_MILLIS), Offset.offset(10_000L)); + + long oneWeekFromNowMillis = NowWithOffset.plus(2, ChronoUnit.WEEKS).evaluate().toEpochMilli(); + assertThat(oneWeekFromNowMillis).isCloseTo(now + (14 * DAY_IN_MILLIS), Offset.offset(10_000L)); + + long oneMonthAgoMillis = NowWithOffset.minus(1, ChronoUnit.MONTHS).evaluate().toEpochMilli(); + assertThat(oneMonthAgoMillis).isCloseTo(now - (30 * DAY_IN_MILLIS), Offset.offset(10_000L + 2 * DAY_IN_MILLIS)); + + long oneMonthFromNowMillis = NowWithOffset.plus(2, ChronoUnit.MONTHS).evaluate().toEpochMilli(); + assertThat(oneMonthFromNowMillis).isCloseTo(now + (60 * DAY_IN_MILLIS), Offset.offset(10_000L + 3 * DAY_IN_MILLIS)); + + long oneYearAgoMillis = NowWithOffset.minus(1, ChronoUnit.YEARS).evaluate().toEpochMilli(); + assertThat(oneYearAgoMillis).isCloseTo(now - (365 * DAY_IN_MILLIS), Offset.offset(10_000L + 2 * DAY_IN_MILLIS)); + + long oneYearFromNowMillis = NowWithOffset.plus(2, ChronoUnit.YEARS).evaluate().toEpochMilli(); + assertThat(oneYearFromNowMillis).isCloseTo(now + (730 * DAY_IN_MILLIS), Offset.offset(10_000L + 3 * DAY_IN_MILLIS)); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializerTest.java new file mode 100644 index 00000000..42f7c3da --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializerTest.java @@ -0,0 +1,93 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.tables.query.serialization; + + +import java.io.IOException; +import java.time.temporal.ChronoUnit; +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.Now; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.EQUALS; +import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.GREATER_THAN; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for QFilterCriteriaDeserializer + *******************************************************************************/ +class QFilterCriteriaDeserializerTest extends BaseTest +{ + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDeserialize() throws IOException + { + { + QFilterCriteria criteria = JsonUtils.toObject(""" + {"fieldName": "id", "operator": "EQUALS", "values": [1]} + """, QFilterCriteria.class); + assertEquals("id", criteria.getFieldName()); + assertEquals(EQUALS, criteria.getOperator()); + assertEquals(List.of(1), criteria.getValues()); + } + + { + QFilterCriteria criteria = JsonUtils.toObject(""" + {"fieldName": "createDate", "operator": "GREATER_THAN", "expression": + {"type": "NowWithOffset", "operator": "PLUS", "amount": 5, "timeUnit": "MINUTES"} + } + """, QFilterCriteria.class); + assertEquals("createDate", criteria.getFieldName()); + assertEquals(GREATER_THAN, criteria.getOperator()); + assertNull(criteria.getValues()); + AbstractFilterExpression expression = criteria.getExpression(); + assertThat(expression).isInstanceOf(NowWithOffset.class); + NowWithOffset nowWithOffset = (NowWithOffset) expression; + assertEquals(5, nowWithOffset.getAmount()); + assertEquals(NowWithOffset.Operator.PLUS, nowWithOffset.getOperator()); + assertEquals(ChronoUnit.MINUTES, nowWithOffset.getTimeUnit()); + } + + { + QFilterCriteria criteria = JsonUtils.toObject(""" + {"fieldName": "orderDate", "operator": "EQUALS", "expression": {"type": "Now"} } + """, QFilterCriteria.class); + assertEquals("orderDate", criteria.getFieldName()); + assertEquals(EQUALS, criteria.getOperator()); + assertNull(criteria.getValues()); + AbstractFilterExpression expression = criteria.getExpression(); + assertThat(expression).isInstanceOf(Now.class); + } + + } +} \ No newline at end of file From b924fdcffa2ce5de6cea8524c6760e6cd6ea841e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jul 2023 21:56:58 +0000 Subject: [PATCH 02/24] Bump h2 from 2.1.210 to 2.2.220 in /qqq-middleware-picocli Bumps [h2](https://github.com/h2database/h2database) from 2.1.210 to 2.2.220. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.1.210...version-2.2.220) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- qqq-middleware-picocli/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-middleware-picocli/pom.xml b/qqq-middleware-picocli/pom.xml index 4eecb082..1aa8f435 100644 --- a/qqq-middleware-picocli/pom.xml +++ b/qqq-middleware-picocli/pom.xml @@ -64,7 +64,7 @@ com.h2database h2 - 2.1.210 + 2.2.220 test From 51dd0b6b298cef539d6327069be1aa766ad60269 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jul 2023 21:57:48 +0000 Subject: [PATCH 03/24] Bump h2 from 2.1.212 to 2.2.220 in /qqq-sample-project Bumps [h2](https://github.com/h2database/h2database) from 2.1.212 to 2.2.220. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.1.212...version-2.2.220) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- qqq-sample-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-sample-project/pom.xml b/qqq-sample-project/pom.xml index e1d80a71..5fdd4cd2 100644 --- a/qqq-sample-project/pom.xml +++ b/qqq-sample-project/pom.xml @@ -68,7 +68,7 @@ com.h2database h2 - 2.1.212 + 2.2.220 test From ca560c933d52442ad7e7ed93bc364cad379ef343 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jul 2023 21:59:09 +0000 Subject: [PATCH 04/24] Bump h2 from 2.1.214 to 2.2.220 in /qqq-backend-module-rdbms Bumps [h2](https://github.com/h2database/h2database) from 2.1.214 to 2.2.220. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.1.214...version-2.2.220) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- qqq-backend-module-rdbms/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-module-rdbms/pom.xml b/qqq-backend-module-rdbms/pom.xml index 4cee90bb..19d778c3 100644 --- a/qqq-backend-module-rdbms/pom.xml +++ b/qqq-backend-module-rdbms/pom.xml @@ -53,7 +53,7 @@ com.h2database h2 - 2.1.214 + 2.2.220 test From 593c9f25f90a613cc5ddc826d0a2ea83bd5ee488 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jul 2023 21:59:44 +0000 Subject: [PATCH 05/24] Bump h2 from 2.1.214 to 2.2.220 in /qqq-middleware-javalin Bumps [h2](https://github.com/h2database/h2database) from 2.1.214 to 2.2.220. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.1.214...version-2.2.220) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- qqq-middleware-javalin/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-middleware-javalin/pom.xml b/qqq-middleware-javalin/pom.xml index ef416c3a..309acce7 100644 --- a/qqq-middleware-javalin/pom.xml +++ b/qqq-middleware-javalin/pom.xml @@ -71,7 +71,7 @@ com.h2database h2 - 2.1.214 + 2.2.220 test From 5cfcb420d09f6b0e778b2ba0672ad8208e679b49 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 10 Jul 2023 09:46:13 -0500 Subject: [PATCH 06/24] CE-535 Initial checkin --- .../module/api/actions/APIRecordUtils.java | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIRecordUtils.java diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIRecordUtils.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIRecordUtils.java new file mode 100644 index 00000000..3f1d4ec9 --- /dev/null +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIRecordUtils.java @@ -0,0 +1,136 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.api.actions; + + +import java.io.Serializable; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class APIRecordUtils +{ + private static final QLogger LOG = QLogger.getLogger(APIRecordUtils.class); + + + + /******************************************************************************* + ** Take a QRecord whose field names are formatted in JSONQuery-style + ** (e.g., 'key' or 'key.subKey' or 'key[index].subKey') + ** and convert it to a JSONObject. + *******************************************************************************/ + public static JSONObject jsonQueryStyleQRecordToJSONObject(QTableMetaData table, QRecord record, boolean includeNonTableFields) + { + try + { + JSONObject body = new JSONObject(); + for(Map.Entry entry : record.getValues().entrySet()) + { + String fieldName = entry.getKey(); + Serializable value = entry.getValue(); + + if(fieldName.contains(".")) + { + JSONObject tmp = body; + String[] parts = fieldName.split("\\."); + for(int i = 0; i < parts.length - 1; i++) + { + String thisPart = parts[i]; + if(thisPart.contains("[")) + { + String arrayName = thisPart.replaceFirst("\\[.*", ""); + if(!tmp.has(arrayName)) + { + tmp.put(arrayName, new JSONArray()); + } + + JSONArray array = tmp.getJSONArray(arrayName); + Integer arrayIndex = Integer.parseInt(thisPart.replaceFirst(".*\\[", "").replaceFirst("].*", "")); + if(array.opt(arrayIndex) == null) + { + array.put(arrayIndex, new JSONObject()); + } + tmp = array.getJSONObject(arrayIndex); + } + else + { + if(!tmp.has(thisPart)) + { + tmp.put(thisPart, new JSONObject()); + } + tmp = tmp.getJSONObject(thisPart); + } + } + tmp.put(parts[parts.length - 1], value); + } + else + { + try + { + QFieldMetaData field = table.getField(fieldName); + body.put(getFieldBackendName(field), value); + } + catch(Exception e) + { + if(includeNonTableFields) + { + LOG.debug("Putting non-table-field in record", logPair("name", fieldName)); + body.put(fieldName, value); + } + } + } + } + return body; + } + catch(Exception e) + { + throw (new QRuntimeException("Error converting record to JSON Object", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static String getFieldBackendName(QFieldMetaData field) + { + String backendName = field.getBackendName(); + if(!StringUtils.hasContent(backendName)) + { + backendName = field.getName(); + } + return (backendName); + } + +} From a943628e84571ad287374531d0e742ab645542c8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 10 Jul 2023 09:47:50 -0500 Subject: [PATCH 07/24] Update to flush buffered pipes - fixes issue where static data supplier records may not appear --- .../qqq/backend/core/actions/async/AsyncRecordPipeLoop.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java index 03d10708..22d73fb5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.actions.async; import java.io.Serializable; import java.util.Optional; import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.actions.reporting.BufferedRecordPipe; import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; @@ -142,6 +143,11 @@ public class AsyncRecordPipeLoop jobState = asyncJobStatus.getState(); } + if(recordPipe instanceof BufferedRecordPipe bufferedRecordPipe) + { + bufferedRecordPipe.finalFlush(); + } + LOG.debug("Job [" + jobUUID + "][" + jobName + "] completed with status: " + asyncJobStatus); /////////////////////////////////// From ed60ad2a96b0b685a152144e25a5b70c13868a93 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 10 Jul 2023 09:49:11 -0500 Subject: [PATCH 08/24] Add doCopySourcePrimaryKeyToCache option - to, copy primary keys from source to cache; apply this in qqq-table cache --- .../actions/tables/helpers/CacheUtils.java | 7 ++-- .../helpers/QueryActionCacheHelper.java | 22 ++++++------- .../metadata/tables/cache/CacheUseCase.java | 32 +++++++++++++++++++ .../tables/QQQTablesMetaDataProvider.java | 1 + 4 files changed, 49 insertions(+), 13 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/CacheUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/CacheUtils.java index a54dd92e..f8e64ff7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/CacheUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/CacheUtils.java @@ -46,7 +46,7 @@ public class CacheUtils /******************************************************************************* ** *******************************************************************************/ - static QRecord mapSourceRecordToCacheRecord(QTableMetaData table, QRecord recordFromSource) + static QRecord mapSourceRecordToCacheRecord(QTableMetaData table, QRecord recordFromSource, CacheUseCase cacheUseCase) { QRecord cacheRecord = new QRecord(recordFromSource); @@ -58,7 +58,10 @@ public class CacheUtils { if(fieldName.equals(table.getPrimaryKeyField())) { - cacheRecord.removeValue(fieldName); + if(!cacheUseCase.getDoCopySourcePrimaryKeyToCache()) + { + cacheRecord.removeValue(fieldName); + } } else if(!cacheRecord.getValues().containsKey(fieldName)) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryActionCacheHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryActionCacheHelper.java index e8b79d6f..2d830636 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryActionCacheHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryActionCacheHelper.java @@ -79,9 +79,9 @@ public class QueryActionCacheHelper { private static final QLogger LOG = QLogger.getLogger(QueryActionCacheHelper.class); - private boolean isQueryInputCacheable = false; - private Set cacheUseCases = new HashSet<>(); - private CacheUseCase.Type activeCacheUseCase = null; + private boolean isQueryInputCacheable = false; + private Map cacheUseCaseMap = new HashMap<>(); + private CacheUseCase activeCacheUseCase = null; private UniqueKey cacheUniqueKey = null; private ListingHash uniqueKeyValues = new ListingHash<>(); @@ -122,7 +122,7 @@ public class QueryActionCacheHelper QTableMetaData table = queryInput.getTable(); List recordsToReturn = recordsFromSource.stream() - .map(r -> CacheUtils.mapSourceRecordToCacheRecord(table, r)) + .map(r -> CacheUtils.mapSourceRecordToCacheRecord(table, r, activeCacheUseCase)) .toList(); queryOutput.addRecords(recordsToReturn); @@ -220,7 +220,7 @@ public class QueryActionCacheHelper List refreshedRecordsToReturn = recordsFromSource.stream() .map(r -> { - QRecord recordToCache = CacheUtils.mapSourceRecordToCacheRecord(table, r); + QRecord recordToCache = CacheUtils.mapSourceRecordToCacheRecord(table, r, activeCacheUseCase); recordToCache.setValue(table.getPrimaryKeyField(), uniqueKeyToPrimaryKeyMap.get(getUniqueKeyValues(recordToCache))); return (recordToCache); }) @@ -391,10 +391,10 @@ public class QueryActionCacheHelper for(CacheUseCase cacheUseCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases())) { - cacheUseCases.add(cacheUseCase.getType()); + cacheUseCaseMap.put(cacheUseCase.getType(), cacheUseCase); } - if(cacheUseCases.contains(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY)) + if(cacheUseCaseMap.containsKey(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY)) { if(queryInput.getFilter() == null) { @@ -475,7 +475,7 @@ public class QueryActionCacheHelper return; } - LOG.trace("Unable to cache: No supported use case: " + cacheUseCases); + LOG.trace("Unable to cache: No supported use case: " + cacheUseCaseMap.keySet()); } @@ -491,7 +491,7 @@ public class QueryActionCacheHelper { this.cacheUniqueKey = uniqueKey; isQueryInputCacheable = true; - activeCacheUseCase = CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY; + activeCacheUseCase = cacheUseCaseMap.get(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY); return true; } } @@ -536,14 +536,14 @@ public class QueryActionCacheHelper List recordsFromSource = null; QTableMetaData table = queryInput.getTable(); - if(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY.equals(activeCacheUseCase)) + if(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY.equals(activeCacheUseCase.getType())) { recordsFromSource = getFromCachedSourceForUniqueKeyToUniqueKey(queryInput, uniqueKeyValues, table.getCacheOf().getSourceTable()); } else { // todo!! - throw (new NotImplementedException("Not-yet-implemented cache use case type: " + activeCacheUseCase)); + throw (new NotImplementedException("Not-yet-implemented cache use case type: " + activeCacheUseCase.getType())); } return (recordsFromSource); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/cache/CacheUseCase.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/cache/CacheUseCase.java index 1aea5d49..7f5852fb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/cache/CacheUseCase.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/cache/CacheUseCase.java @@ -49,6 +49,7 @@ public class CacheUseCase ////////////////////////// private UniqueKey cacheUniqueKey; private UniqueKey sourceUniqueKey; + private boolean doCopySourcePrimaryKeyToCache = false; private List excludeRecordsMatching; @@ -222,4 +223,35 @@ public class CacheUseCase return (this); } + + + /******************************************************************************* + ** Getter for doCopySourcePrimaryKeyToCache + *******************************************************************************/ + public boolean getDoCopySourcePrimaryKeyToCache() + { + return (this.doCopySourcePrimaryKeyToCache); + } + + + + /******************************************************************************* + ** Setter for doCopySourcePrimaryKeyToCache + *******************************************************************************/ + public void setDoCopySourcePrimaryKeyToCache(boolean doCopySourcePrimaryKeyToCache) + { + this.doCopySourcePrimaryKeyToCache = doCopySourcePrimaryKeyToCache; + } + + + + /******************************************************************************* + ** Fluent setter for doCopySourcePrimaryKeyToCache + *******************************************************************************/ + public CacheUseCase withDoCopySourcePrimaryKeyToCache(boolean doCopySourcePrimaryKeyToCache) + { + this.doCopySourcePrimaryKeyToCache = doCopySourcePrimaryKeyToCache; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTablesMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTablesMetaDataProvider.java index 1f40e9ec..100d705b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTablesMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTablesMetaDataProvider.java @@ -105,6 +105,7 @@ public class QQQTablesMetaDataProvider .withCacheSourceMisses(false) .withCacheUniqueKey(new UniqueKey("name")) .withSourceUniqueKey(new UniqueKey("name")) + .withDoCopySourcePrimaryKeyToCache(true) ) ); From e6816174c3fc668416193e0bd3fd137af5e26e97 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 10 Jul 2023 11:07:06 -0500 Subject: [PATCH 09/24] Test for APIRecordUtils.jsonQueryStyleQRecordToJSONObject --- .../api/actions/APIRecordUtilsTest.java | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/APIRecordUtilsTest.java diff --git a/qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/APIRecordUtilsTest.java b/qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/APIRecordUtilsTest.java new file mode 100644 index 00000000..90eb8e8f --- /dev/null +++ b/qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/APIRecordUtilsTest.java @@ -0,0 +1,79 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.api.actions; + + +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.api.BaseTest; +import com.kingsrook.qqq.backend.module.api.TestUtils; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + + +/******************************************************************************* + ** Unit test for APIRecordUtils + *******************************************************************************/ +class APIRecordUtilsTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + QTableMetaData table = QContext.getQInstance().getTable(TestUtils.MOCK_TABLE_NAME); + table.withField(new QFieldMetaData("myField", QFieldType.INTEGER).withBackendName("myBackendName")); + + QRecord record = new QRecord() + .withValue("foo", 1) + .withValue("bar.baz", 2) + .withValue("list[0].a", 3) + .withValue("list[1].a", 4) + .withValue("list[1].b", 5) + .withValue("myField", 6); + + JSONObject jsonObject = APIRecordUtils.jsonQueryStyleQRecordToJSONObject(table, record, true); + assertEquals(1, jsonObject.getInt("foo")); + assertEquals(2, jsonObject.getJSONObject("bar").getInt("baz")); + assertEquals(3, jsonObject.getJSONArray("list").getJSONObject(0).getInt("a")); + assertEquals(4, jsonObject.getJSONArray("list").getJSONObject(1).getInt("a")); + assertEquals(5, jsonObject.getJSONArray("list").getJSONObject(1).getInt("b")); + assertEquals(6, jsonObject.getInt("myBackendName")); + assertFalse(jsonObject.has("myField")); + + /////////////////////////////////////////////////////// + // if we say "false" for includeNonTableFields, then // + // we should only get the myField field // + /////////////////////////////////////////////////////// + jsonObject = APIRecordUtils.jsonQueryStyleQRecordToJSONObject(table, record, false); + assertFalse(jsonObject.has("foo")); + assertEquals(6, jsonObject.getInt("myBackendName")); + } + +} \ No newline at end of file From 086787a5ca4998470b269712b5763c5a50981f1a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 11 Jul 2023 08:31:15 -0500 Subject: [PATCH 10/24] CE-535 Cleanup in DeleteAction - add omitDmlAudit & auditContext; be more sure not to delete associations if errors --- .../core/actions/tables/DeleteAction.java | 51 ++++++++--- .../actions/tables/count/CountInput.java | 22 +++++ .../actions/tables/delete/DeleteInput.java | 65 +++++++++++++ .../core/actions/tables/DeleteActionTest.java | 91 +++++++++++++++++++ 4 files changed, 216 insertions(+), 13 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java index 5e30832b..ca9badc9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java @@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.LogPair; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.audits.DMLAuditInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; @@ -117,12 +118,14 @@ public class DeleteAction ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if there's a query filter, but the interface doesn't support using a query filter, then do a query for the filter, to get a list of primary keys instead // + // or - anytime there are associations on the table we want primary keys, as that's what the manage associations method uses // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(deleteInput.getQueryFilter() != null && !deleteInterface.supportsQueryFilterInput()) + if(deleteInput.getQueryFilter() != null && (!deleteInterface.supportsQueryFilterInput() || CollectionUtils.nullSafeHasContents(table.getAssociations()))) { - LOG.info("Querying for primary keys, for backend module " + qModule.getBackendType() + " which does not support queryFilter input for deletes"); + LOG.info("Querying for primary keys, for table " + table.getName() + " in backend module " + qModule.getBackendType() + " which does not support queryFilter input for deletes (or the table has associations)"); List primaryKeyList = getPrimaryKeysFromQueryFilter(deleteInput); deleteInput.setPrimaryKeys(primaryKeyList); + primaryKeys = primaryKeyList; if(primaryKeyList.isEmpty()) { @@ -165,10 +168,22 @@ public class DeleteAction if(!primaryKeysToRemoveFromInput.isEmpty()) { - primaryKeys.removeAll(primaryKeysToRemoveFromInput); + if(primaryKeys == null) + { + LOG.warn("There were primary keys to remove from the input, but no primary key list (filter supplied as input?)", new LogPair("primaryKeysToRemoveFromInput", primaryKeysToRemoveFromInput)); + } + else + { + primaryKeys.removeAll(primaryKeysToRemoveFromInput); + } } } + //////////////////////////////////////////////////////////////////////////////////////////////// + // stash a copy of primary keys that didn't have errors (for use in manageAssociations below) // + //////////////////////////////////////////////////////////////////////////////////////////////// + Set primaryKeysWithoutErrors = new HashSet<>(CollectionUtils.nonNullList(primaryKeys)); + //////////////////////////////////// // have the backend do the delete // //////////////////////////////////// @@ -187,11 +202,13 @@ public class DeleteAction /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if a record had a validation warning, but then an execution error, remove it from the warning list - so it's only in one of them. // + // also, always remove from /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// for(QRecord outputRecordWithError : outputRecordsWithErrors) { Serializable pkey = outputRecordWithError.getValue(primaryKeyFieldName); recordsWithValidationWarnings.remove(pkey); + primaryKeysWithoutErrors.remove(pkey); } /////////////////////////////////////////////////////////////////////////////////////////// @@ -211,15 +228,23 @@ public class DeleteAction //////////////////////////////////////// // delete associations, if applicable // //////////////////////////////////////// - manageAssociations(deleteInput); + manageAssociations(primaryKeysWithoutErrors, deleteInput); - /////////////////////////////////// - // do the audit // - // todo - add input.omitDmlAudit // - /////////////////////////////////// - DMLAuditInput dmlAuditInput = new DMLAuditInput().withTableActionInput(deleteInput); - oldRecordList.ifPresent(l -> dmlAuditInput.setRecordList(l)); - new DMLAuditAction().execute(dmlAuditInput); + ////////////////// + // do the audit // + ////////////////// + if(deleteInput.getOmitDmlAudit()) + { + LOG.debug("Requested to omit DML audit"); + } + else + { + DMLAuditInput dmlAuditInput = new DMLAuditInput() + .withTableActionInput(deleteInput) + .withAuditContext(deleteInput.getAuditContext()); + oldRecordList.ifPresent(l -> dmlAuditInput.setRecordList(l)); + new DMLAuditAction().execute(dmlAuditInput); + } ////////////////////////////////////////////////////////////// // finally, run the post-delete customizer, if there is one // @@ -340,7 +365,7 @@ public class DeleteAction /******************************************************************************* ** *******************************************************************************/ - private void manageAssociations(DeleteInput deleteInput) throws QException + private void manageAssociations(Set primaryKeysWithoutErrors, DeleteInput deleteInput) throws QException { QTableMetaData table = deleteInput.getTable(); for(Association association : CollectionUtils.nonNullList(table.getAssociations())) @@ -353,7 +378,7 @@ public class DeleteAction if(join.getJoinOns().size() == 1 && join.getJoinOns().get(0).getLeftField().equals(table.getPrimaryKeyField())) { - filter.addCriteria(new QFilterCriteria(join.getJoinOns().get(0).getRightField(), QCriteriaOperator.IN, deleteInput.getPrimaryKeys())); + filter.addCriteria(new QFilterCriteria(join.getJoinOns().get(0).getRightField(), QCriteriaOperator.IN, new ArrayList<>(primaryKeysWithoutErrors))); } else { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountInput.java index 1a4863e5..b6383d99 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountInput.java @@ -51,6 +51,17 @@ public class CountInput extends AbstractTableActionInput + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public CountInput(String tableName) + { + setTableName(tableName); + } + + + /******************************************************************************* ** Getter for filter ** @@ -152,4 +163,15 @@ public class CountInput extends AbstractTableActionInput return (this); } + + + /******************************************************************************* + ** Fluent setter for filter + *******************************************************************************/ + public CountInput withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java index c39ed3ec..3945246e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java @@ -43,6 +43,9 @@ public class DeleteInput extends AbstractTableActionInput private QQueryFilter queryFilter; private InputSource inputSource = QInputSource.SYSTEM; + private boolean omitDmlAudit = false; + private String auditContext = null; + /******************************************************************************* @@ -211,4 +214,66 @@ public class DeleteInput extends AbstractTableActionInput return (this); } + + + /******************************************************************************* + ** Getter for omitDmlAudit + *******************************************************************************/ + public boolean getOmitDmlAudit() + { + return (this.omitDmlAudit); + } + + + + /******************************************************************************* + ** Setter for omitDmlAudit + *******************************************************************************/ + public void setOmitDmlAudit(boolean omitDmlAudit) + { + this.omitDmlAudit = omitDmlAudit; + } + + + + /******************************************************************************* + ** Fluent setter for omitDmlAudit + *******************************************************************************/ + public DeleteInput withOmitDmlAudit(boolean omitDmlAudit) + { + this.omitDmlAudit = omitDmlAudit; + return (this); + } + + + + /******************************************************************************* + ** Getter for auditContext + *******************************************************************************/ + public String getAuditContext() + { + return (this.auditContext); + } + + + + /******************************************************************************* + ** Setter for auditContext + *******************************************************************************/ + public void setAuditContext(String auditContext) + { + this.auditContext = auditContext; + } + + + + /******************************************************************************* + ** Fluent setter for auditContext + *******************************************************************************/ + public DeleteInput withAuditContext(String auditContext) + { + this.auditContext = auditContext; + return (this); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteActionTest.java index 8492eff3..076a3f39 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteActionTest.java @@ -25,11 +25,15 @@ package com.kingsrook.qqq.backend.core.actions.tables; import java.util.List; import java.util.Objects; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreDeleteCustomizer; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; @@ -41,6 +45,10 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; import com.kingsrook.qqq.backend.core.utils.TestUtils; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; @@ -399,4 +407,87 @@ class DeleteActionTest extends BaseTest new InsertAction().execute(insertInput); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDeleteWithErrorsDoesntDeleteAssociations() throws QException + { + QContext.getQSession().setSecurityKeyValues(MapBuilder.of(TestUtils.SECURITY_KEY_TYPE_STORE, ListBuilder.of(1))); + + QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + table.withCustomizer(TableCustomizers.PRE_DELETE_RECORD, new QCodeReference(OrderPreDeleteCustomizer.class)); + + //////////////////////////////////////////////////////////////////////////////////////////////// + // insert 2 orders - one that will fail to delete, and one that will warn, but should delete. // + //////////////////////////////////////////////////////////////////////////////////////////////// + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_ORDER) + .withRecords(List.of( + new QRecord().withValue("id", OrderPreDeleteCustomizer.DELETE_ERROR_ID).withValue("storeId", 1).withValue("orderNo", "ORD123") + .withAssociatedRecord("orderLine", new QRecord().withValue("sku", "BASIC1").withValue("quantity", 1) + .withAssociatedRecord("extrinsics", new QRecord().withValue("key", "LINE-EXT-1.1").withValue("value", "LINE-VAL-1"))), + + new QRecord().withValue("id", OrderPreDeleteCustomizer.DELETE_WARN_ID).withValue("storeId", 1).withValue("orderNo", "ORD124") + .withAssociatedRecord("orderLine", new QRecord().withValue("sku", "BASIC3").withValue("quantity", 3)) + .withAssociatedRecord("extrinsics", new QRecord().withValue("key", "YOUR-FIELD-1").withValue("value", "YOUR-VALUE-1")) + ))); + + /////////////////////////// + // confirm insert counts // + /////////////////////////// + assertEquals(2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER)).getCount()); + assertEquals(2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_LINE_ITEM)).getCount()); + assertEquals(1, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC)).getCount()); + assertEquals(1, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER_EXTRINSIC)).getCount()); + + ///////////////////////////// + // try to delete them both // + ///////////////////////////// + new DeleteAction().execute(new DeleteInput(TestUtils.TABLE_NAME_ORDER).withPrimaryKeys(List.of(OrderPreDeleteCustomizer.DELETE_WARN_ID, OrderPreDeleteCustomizer.DELETE_WARN_ID))); + + /////////////////////// + // count what's left // + /////////////////////// + assertEquals(1, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER)).getCount()); + assertEquals(1, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_LINE_ITEM)).getCount()); + assertEquals(1, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC)).getCount()); + assertEquals(0, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER_EXTRINSIC)).getCount()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class OrderPreDeleteCustomizer extends AbstractPreDeleteCustomizer + { + public static final Integer DELETE_ERROR_ID = 9999; + public static final Integer DELETE_WARN_ID = 9998; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List apply(List records) + { + for(QRecord record : records) + { + if(DELETE_ERROR_ID.equals(record.getValue("id"))) + { + record.addError(new BadInputStatusMessage("You may not delete this order")); + } + else if(DELETE_WARN_ID.equals(record.getValue("id"))) + { + record.addWarning(new QWarningMessage("It was bad that you deleted this order")); + } + } + + return (records); + } + } + } From 953d97c554a4d02b868d78297bdd0d0831cef6b2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 11 Jul 2023 08:31:28 -0500 Subject: [PATCH 11/24] CE-535 add auditContext --- .../core/actions/tables/InsertAction.java | 5 ++- .../actions/tables/insert/InsertInput.java | 32 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index 62871c8e..0b5adcb0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -155,7 +155,10 @@ public class InsertAction extends AbstractQActionFunction Date: Tue, 11 Jul 2023 08:40:25 -0500 Subject: [PATCH 12/24] CE-535 new replace action, with test --- .../core/actions/tables/ReplaceAction.java | 174 ++++++++++ .../actions/tables/replace/ReplaceInput.java | 210 ++++++++++++ .../actions/tables/replace/ReplaceOutput.java | 133 ++++++++ .../actions/tables/ReplaceActionTest.java | 300 ++++++++++++++++++ 4 files changed, 817 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceOutput.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java new file mode 100644 index 00000000..6f5c441b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java @@ -0,0 +1,174 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.tables; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.replace.ReplaceInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.replace.ReplaceOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** Action to do a "replace" - e.g: Update rows with unique-key values that are + ** already in the table; insert rows whose unique keys weren't already in the + ** table, and delete rows that weren't in the input (all based on a + ** UniqueKey that's part of the input) + ** + ** Note - the filter in the ReplaceInput - its role is to limit what rows are + ** potentially deleted. e.g., if you have a table that's segmented, and you're + ** only replacing a particular segment of it (say, for 1 client), then you pass + ** in a filter that finds rows matching that segment. See Test for example. + *******************************************************************************/ +public class ReplaceAction extends AbstractQActionFunction +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public ReplaceOutput execute(ReplaceInput input) throws QException + { + ReplaceOutput output = new ReplaceOutput(); + + QBackendTransaction transaction = input.getTransaction(); + boolean weOwnTheTransaction = false; + + try + { + QTableMetaData table = input.getTable(); + UniqueKey uniqueKey = input.getKey(); + String primaryKeyField = table.getPrimaryKeyField(); + if(transaction == null) + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(input.getTableName()); + transaction = new InsertAction().openTransaction(insertInput); + weOwnTheTransaction = true; + } + + List insertList = new ArrayList<>(); + List updateList = new ArrayList<>(); + List primaryKeysToKeep = new ArrayList<>(); + + for(List page : CollectionUtils.getPages(input.getRecords(), 1000)) + { + /////////////////////////////////////////////////////////////////////////////////// + // originally it was thought that we'd need to pass the filter in here // + // but, it's been decided not to. the filter only applies to what we can delete // + /////////////////////////////////////////////////////////////////////////////////// + Map, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey); + for(QRecord record : page) + { + Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); + if(keyValues.isPresent()) + { + if(existingKeys.containsKey(keyValues.get())) + { + Serializable primaryKey = existingKeys.get(keyValues.get()); + record.setValue(primaryKeyField, primaryKey); + updateList.add(record); + primaryKeysToKeep.add(primaryKey); + } + else + { + insertList.add(record); + } + } + } + } + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(table.getName()); + insertInput.setRecords(insertList); + insertInput.setTransaction(transaction); + insertInput.setOmitDmlAudit(input.getOmitDmlAudit()); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + primaryKeysToKeep.addAll(insertOutput.getRecords().stream().map(r -> r.getValue(primaryKeyField)).toList()); + output.setInsertOutput(insertOutput); + + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(table.getName()); + updateInput.setRecords(updateList); + updateInput.setTransaction(transaction); + updateInput.setOmitDmlAudit(input.getOmitDmlAudit()); + UpdateOutput updateOutput = new UpdateAction().execute(updateInput); + output.setUpdateOutput(updateOutput); + + QQueryFilter deleteFilter = new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.NOT_IN, primaryKeysToKeep)); + if(input.getFilter() != null) + { + deleteFilter.addSubFilter(input.getFilter()); + } + + DeleteInput deleteInput = new DeleteInput(); + deleteInput.setTableName(table.getName()); + deleteInput.setQueryFilter(deleteFilter); + deleteInput.setTransaction(transaction); + deleteInput.setOmitDmlAudit(input.getOmitDmlAudit()); + DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput); + output.setDeleteOutput(deleteOutput); + + if(weOwnTheTransaction) + { + transaction.commit(); + } + + return (output); + } + catch(Exception e) + { + if(weOwnTheTransaction) + { + transaction.rollback(); + } + throw (new QException("Error executing replace action", e)); + } + finally + { + if(weOwnTheTransaction) + { + transaction.close(); + } + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java new file mode 100644 index 00000000..fe09cf5e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java @@ -0,0 +1,210 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.tables.replace; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ReplaceInput extends AbstractTableActionInput +{ + private QBackendTransaction transaction; + private UniqueKey key; + private List records; + private QQueryFilter filter; + + private boolean omitDmlAudit = false; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ReplaceInput() + { + } + + + + /******************************************************************************* + ** Getter for transaction + *******************************************************************************/ + public QBackendTransaction getTransaction() + { + return (this.transaction); + } + + + + /******************************************************************************* + ** Setter for transaction + *******************************************************************************/ + public void setTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + } + + + + /******************************************************************************* + ** Fluent setter for transaction + *******************************************************************************/ + public ReplaceInput withTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + return (this); + } + + + + /******************************************************************************* + ** Getter for records + *******************************************************************************/ + public List getRecords() + { + return (this.records); + } + + + + /******************************************************************************* + ** Setter for records + *******************************************************************************/ + public void setRecords(List records) + { + this.records = records; + } + + + + /******************************************************************************* + ** Fluent setter for records + *******************************************************************************/ + public ReplaceInput withRecords(List records) + { + this.records = records; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter + *******************************************************************************/ + public QQueryFilter getFilter() + { + return (this.filter); + } + + + + /******************************************************************************* + ** Setter for filter + *******************************************************************************/ + public void setFilter(QQueryFilter filter) + { + this.filter = filter; + } + + + + /******************************************************************************* + ** Fluent setter for filter + *******************************************************************************/ + public ReplaceInput withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + + + /******************************************************************************* + ** Getter for key + *******************************************************************************/ + public UniqueKey getKey() + { + return (this.key); + } + + + + /******************************************************************************* + ** Setter for key + *******************************************************************************/ + public void setKey(UniqueKey key) + { + this.key = key; + } + + + + /******************************************************************************* + ** Fluent setter for key + *******************************************************************************/ + public ReplaceInput withKey(UniqueKey key) + { + this.key = key; + return (this); + } + + + + /******************************************************************************* + ** Getter for omitDmlAudit + *******************************************************************************/ + public boolean getOmitDmlAudit() + { + return (this.omitDmlAudit); + } + + + + /******************************************************************************* + ** Setter for omitDmlAudit + *******************************************************************************/ + public void setOmitDmlAudit(boolean omitDmlAudit) + { + this.omitDmlAudit = omitDmlAudit; + } + + + + /******************************************************************************* + ** Fluent setter for omitDmlAudit + *******************************************************************************/ + public ReplaceInput withOmitDmlAudit(boolean omitDmlAudit) + { + this.omitDmlAudit = omitDmlAudit; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceOutput.java new file mode 100644 index 00000000..95c6f649 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceOutput.java @@ -0,0 +1,133 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.tables.replace; + + +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ReplaceOutput extends AbstractActionOutput +{ + private InsertOutput insertOutput; + private UpdateOutput updateOutput; + private DeleteOutput deleteOutput; + + + + /******************************************************************************* + ** Getter for insertOutput + *******************************************************************************/ + public InsertOutput getInsertOutput() + { + return (this.insertOutput); + } + + + + /******************************************************************************* + ** Setter for insertOutput + *******************************************************************************/ + public void setInsertOutput(InsertOutput insertOutput) + { + this.insertOutput = insertOutput; + } + + + + /******************************************************************************* + ** Fluent setter for insertOutput + *******************************************************************************/ + public ReplaceOutput withInsertOutput(InsertOutput insertOutput) + { + this.insertOutput = insertOutput; + return (this); + } + + + + /******************************************************************************* + ** Getter for updateOutput + *******************************************************************************/ + public UpdateOutput getUpdateOutput() + { + return (this.updateOutput); + } + + + + /******************************************************************************* + ** Setter for updateOutput + *******************************************************************************/ + public void setUpdateOutput(UpdateOutput updateOutput) + { + this.updateOutput = updateOutput; + } + + + + /******************************************************************************* + ** Fluent setter for updateOutput + *******************************************************************************/ + public ReplaceOutput withUpdateOutput(UpdateOutput updateOutput) + { + this.updateOutput = updateOutput; + return (this); + } + + + + /******************************************************************************* + ** Getter for deleteOutput + *******************************************************************************/ + public DeleteOutput getDeleteOutput() + { + return (this.deleteOutput); + } + + + + /******************************************************************************* + ** Setter for deleteOutput + *******************************************************************************/ + public void setDeleteOutput(DeleteOutput deleteOutput) + { + this.deleteOutput = deleteOutput; + } + + + + /******************************************************************************* + ** Fluent setter for deleteOutput + *******************************************************************************/ + public ReplaceOutput withDeleteOutput(DeleteOutput deleteOutput) + { + this.deleteOutput = deleteOutput; + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java new file mode 100644 index 00000000..4ce0e8ee --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java @@ -0,0 +1,300 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.tables; + + +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.replace.ReplaceInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.replace.ReplaceOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for ReplaceAction + *******************************************************************************/ +class ReplaceActionTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWithoutFilter() throws QException + { + String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY; + + /////////////////////////////// + // start with these 2 people // + /////////////////////////////// + new InsertAction().execute(new InsertInput(tableName).withRecords(List.of( + new QRecord().withValue("firstName", "Homer").withValue("lastName", "Simpson").withValue("noOfShoes", 1), + new QRecord().withValue("firstName", "Mr.").withValue("lastName", "Burns") + ))); + + assertEquals(1, countByFirstName("Homer")); + assertEquals(1, countByFirstName("Mr.")); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now do a replace - updating one, inserting one, and (since it's not included in the list), deleting the other // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List newPeople = List.of( + new QRecord().withValue("firstName", "Homer").withValue("lastName", "Simpson").withValue("noOfShoes", 2), + new QRecord().withValue("firstName", "Ned").withValue("lastName", "Flanders") + ); + + ReplaceInput replaceInput = new ReplaceInput(); + replaceInput.setTableName(tableName); + replaceInput.setKey(new UniqueKey("firstName", "lastName")); + replaceInput.setOmitDmlAudit(true); + replaceInput.setRecords(newPeople); + replaceInput.setFilter(null); + ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput); + + assertEquals(1, replaceOutput.getInsertOutput().getRecords().size()); + assertEquals(1, replaceOutput.getUpdateOutput().getRecords().size()); + assertEquals(1, replaceOutput.getDeleteOutput().getDeletedRecordCount()); + + ////////////////////////////// + // assert homer was updated // + ////////////////////////////// + assertEquals(1, countByFirstName("Homer")); + assertEquals(2, getNoOfShoes("Homer", "Simpson")); + + /////////////////////////////////// + // assert Mr (burns) was deleted // + /////////////////////////////////// + assertEquals(0, countByFirstName("Mr.")); + + ///////////////////////////// + // assert ned was inserted // + ///////////////////////////// + assertEquals(1, countByFirstName("Ned")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOnlyInsertAndDelete() throws QException + { + String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY; + + /////////////////////////////// + // start with these 2 people // + /////////////////////////////// + new InsertAction().execute(new InsertInput(tableName).withRecords(List.of( + new QRecord().withValue("firstName", "Homer").withValue("lastName", "Simpson").withValue("noOfShoes", 1), + new QRecord().withValue("firstName", "Marge").withValue("lastName", "Simpson").withValue("noOfShoes", 1) + ))); + + ////////////////////////////////////////// + // now do a replace that fully replaces // + ////////////////////////////////////////// + List newPeople = List.of( + new QRecord().withValue("firstName", "Ned").withValue("lastName", "Flanders"), + new QRecord().withValue("firstName", "Maude").withValue("lastName", "Flanders") + ); + + ReplaceInput replaceInput = new ReplaceInput(); + replaceInput.setTableName(tableName); + replaceInput.setKey(new UniqueKey("firstName", "lastName")); + replaceInput.setOmitDmlAudit(true); + replaceInput.setRecords(newPeople); + replaceInput.setFilter(null); + ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput); + + assertEquals(2, replaceOutput.getInsertOutput().getRecords().size()); + assertEquals(0, replaceOutput.getUpdateOutput().getRecords().size()); + assertEquals(2, replaceOutput.getDeleteOutput().getDeletedRecordCount()); + + /////////////////////////////////////// + // assert homer & marge were deleted // + /////////////////////////////////////// + assertEquals(0, countByFirstName("Homer")); + assertEquals(0, countByFirstName("Marge")); + + ////////////////////////////////////// + // assert ned & maude were inserted // + ////////////////////////////////////// + assertEquals(1, countByFirstName("Ned")); + assertEquals(1, countByFirstName("Maude")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOnlyUpdates() throws QException + { + String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY; + + /////////////////////////////// + // start with these 2 people // + /////////////////////////////// + new InsertAction().execute(new InsertInput(tableName).withRecords(List.of( + new QRecord().withValue("firstName", "Homer").withValue("lastName", "Simpson").withValue("noOfShoes", 1), + new QRecord().withValue("firstName", "Marge").withValue("lastName", "Simpson").withValue("noOfShoes", 1) + ))); + + ///////////////////////////////////////////// + // now do a replace that just updates them // + ///////////////////////////////////////////// + List newPeople = List.of( + new QRecord().withValue("firstName", "Homer").withValue("lastName", "Simpson").withValue("noOfShoes", 2), + new QRecord().withValue("firstName", "Marge").withValue("lastName", "Simpson").withValue("noOfShoes", 2) + ); + + ReplaceInput replaceInput = new ReplaceInput(); + replaceInput.setTableName(tableName); + replaceInput.setKey(new UniqueKey("firstName", "lastName")); + replaceInput.setOmitDmlAudit(true); + replaceInput.setRecords(newPeople); + replaceInput.setFilter(null); + ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput); + + assertEquals(0, replaceOutput.getInsertOutput().getRecords().size()); + assertEquals(2, replaceOutput.getUpdateOutput().getRecords().size()); + assertEquals(0, replaceOutput.getDeleteOutput().getDeletedRecordCount()); + + /////////////////////////////////////// + // assert homer & marge were updated // + /////////////////////////////////////// + assertEquals(2, getNoOfShoes("Homer", "Simpson")); + assertEquals(2, getNoOfShoes("Marge", "Simpson")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWithFilter() throws QException + { + String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY; + + ///////////////////////////////////// + // start w/ 3 simpsons and a burns // + ///////////////////////////////////// + new InsertAction().execute(new InsertInput(tableName).withRecords(List.of( + new QRecord().withValue("firstName", "Homer").withValue("lastName", "Simpson").withValue("noOfShoes", 1), + new QRecord().withValue("firstName", "Marge").withValue("lastName", "Simpson").withValue("noOfShoes", 2), + new QRecord().withValue("firstName", "Bart").withValue("lastName", "Simpson").withValue("noOfShoes", 3), + new QRecord().withValue("firstName", "Mr.").withValue("lastName", "Burns") + ))); + + assertEquals(1, countByFirstName("Homer")); + assertEquals(1, countByFirstName("Marge")); + assertEquals(1, countByFirstName("Bart")); + assertEquals(1, countByFirstName("Mr.")); + + ///////////////////////////////////////////////////////////////////////////////// + // now - we'll replace the simpsons only - note the filter in the ReplaceInput // + // so even though Burns isn't in this list, he shouldn't be deleted. // + ///////////////////////////////////////////////////////////////////////////////// + List newPeople = List.of( + new QRecord().withValue("firstName", "Homer").withValue("lastName", "Simpson").withValue("noOfShoes", 4), + new QRecord().withValue("firstName", "Marge").withValue("lastName", "Simpson"), + new QRecord().withValue("firstName", "Lisa").withValue("lastName", "Simpson").withValue("noOfShoes", 5) + ); + + ReplaceInput replaceInput = new ReplaceInput(); + replaceInput.setTableName(tableName); + replaceInput.setKey(new UniqueKey("firstName", "lastName")); + replaceInput.setOmitDmlAudit(true); + replaceInput.setRecords(newPeople); + replaceInput.setFilter(new QQueryFilter(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, "Simpson"))); + ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput); + + assertEquals(1, replaceOutput.getInsertOutput().getRecords().size()); + assertEquals(2, replaceOutput.getUpdateOutput().getRecords().size()); + assertEquals(1, replaceOutput.getDeleteOutput().getDeletedRecordCount()); + + ////////////////////////////// + // assert homer was updated // + ////////////////////////////// + assertEquals(1, countByFirstName("Homer")); + assertEquals(4, getNoOfShoes("Homer", "Simpson")); + + /////////////////////////////// + // assert Marge was no-op'ed // + /////////////////////////////// + assertEquals(1, countByFirstName("Marge")); + assertEquals(2, getNoOfShoes("Marge", "Simpson")); + + //////////////////////////////////// + // assert Mr (burns) was no-op'ed // + //////////////////////////////////// + assertEquals(1, countByFirstName("Mr.")); + assertNull(getNoOfShoes("Mr.", "Burns")); + + ///////////////////////////// + // assert Bart was deleted // + ///////////////////////////// + assertEquals(0, countByFirstName("Bart")); + + ////////////////////////////// + // assert Lisa was inserted // + ////////////////////////////// + assertEquals(1, countByFirstName("Lisa")); + assertEquals(5, getNoOfShoes("Lisa", "Simpson")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Integer countByFirstName(String firstName) throws QException + { + return new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, firstName)))).getCount(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Integer getNoOfShoes(String firstName, String lastName) throws QException + { + return new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withUniqueKey(Map.of("firstName", firstName, "lastName", lastName))).getValueInteger("noOfShoes"); + } + +} \ No newline at end of file From c003d448d6bc08c85b4be34c674ac089432a75d0 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Wed, 12 Jul 2023 21:07:20 -0500 Subject: [PATCH 13/24] updates from last sprint's story --- .../backend/core/actions/processes/RunProcessAction.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index ee21c738..0fee0fcc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -502,9 +502,12 @@ public class RunProcessAction QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getSchedule().getVariantBackend()); if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue())) { - throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'")); + LOG.info("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"); + } + else + { + basepullKeyValue += "-" + session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue()); } - basepullKeyValue += "-" + session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue()); } return (basepullKeyValue); From c04ab42bd901bf19dfac9429e5e41e9a11d7e6cc Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Wed, 12 Jul 2023 21:09:18 -0500 Subject: [PATCH 14/24] CE-534: updates to support direct carrier tracker --- .../model/metadata/queues/QQueueMetaData.java | 15 +++++++++- .../module/api/actions/BaseAPIActionUtil.java | 30 ++++++------------- .../module/api/model/AuthorizationType.java | 1 + 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QQueueMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QQueueMetaData.java index e0822b0d..07e6bf6e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QQueueMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QQueueMetaData.java @@ -22,6 +22,8 @@ package com.kingsrook.qqq.backend.core.model.metadata.queues; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; @@ -34,7 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaDa ** The processName is the code that runs for messages found on the queue. ** The schedule may not be used by all provider types, but defines when the queue is polled. *******************************************************************************/ -public class QQueueMetaData +public class QQueueMetaData implements TopLevelMetaDataInterface { private String name; private String providerName; @@ -213,4 +215,15 @@ public class QQueueMetaData return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void addSelfToInstance(QInstance qInstance) + { + qInstance.addQueue(this); + } + } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 3856bd6b..4f7ec617 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -681,23 +681,13 @@ public class BaseAPIActionUtil /////////////////////////////////////////////////////////////////////////////////////////// switch(backendMetaData.getAuthorizationType()) { - case BASIC_AUTH_API_KEY: - request.addHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getApiKey())); - break; - - case BASIC_AUTH_USERNAME_PASSWORD: - request.addHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getUsername(), backendMetaData.getPassword())); - break; - - case API_KEY_HEADER: - request.addHeader("API-Key", backendMetaData.getApiKey()); - break; - - case OAUTH2: - request.setHeader("Authorization", "Bearer " + getOAuth2Token()); - break; - - case API_KEY_QUERY_PARAM: + case BASIC_AUTH_API_KEY -> request.addHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getApiKey())); + case BASIC_AUTH_USERNAME_PASSWORD -> request.addHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getUsername(), backendMetaData.getPassword())); + case API_KEY_HEADER -> request.addHeader("API-Key", backendMetaData.getApiKey()); + case API_TOKEN -> request.addHeader("Authorization", "Token " + backendMetaData.getApiKey()); + case OAUTH2 -> request.setHeader("Authorization", "Bearer " + getOAuth2Token()); + case API_KEY_QUERY_PARAM -> + { try { String uri = request.getURI().toString(); @@ -709,10 +699,8 @@ public class BaseAPIActionUtil { throw (new QException("Error setting authorization query parameter", e)); } - break; - - default: - throw new IllegalArgumentException("Unexpected authorization type: " + backendMetaData.getAuthorizationType()); + } + default -> throw new IllegalArgumentException("Unexpected authorization type: " + backendMetaData.getAuthorizationType()); } } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/AuthorizationType.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/AuthorizationType.java index 61497df4..7e8060f3 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/AuthorizationType.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/AuthorizationType.java @@ -28,6 +28,7 @@ package com.kingsrook.qqq.backend.module.api.model; public enum AuthorizationType { API_KEY_HEADER, + API_TOKEN, BASIC_AUTH_API_KEY, BASIC_AUTH_USERNAME_PASSWORD, OAUTH2, From 2422d09c314e9cd02830736bf86466d52eaf52c7 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 13 Jul 2023 09:17:07 -0500 Subject: [PATCH 15/24] Stop doing criteria expressions as their own thing, and instead put them in the values list --- .../actions/tables/query/QFilterCriteria.java | 55 ++------------- .../expressions/AbstractFilterExpression.java | 2 +- .../QFilterCriteriaDeserializer.java | 67 ++++++++++--------- .../qqq/backend/core/utils/JsonUtils.java | 38 +++++++++++ .../QFilterCriteriaDeserializerTest.java | 39 ++++++++--- .../rdbms/actions/AbstractRDBMSAction.java | 19 ++++-- .../rdbms/actions/RDBMSQueryActionTest.java | 7 +- 7 files changed, 128 insertions(+), 99 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java index f43f7eb9..0072b6c9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java @@ -28,7 +28,6 @@ import java.util.Arrays; import java.util.List; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression; import com.kingsrook.qqq.backend.core.model.actions.tables.query.serialization.QFilterCriteriaDeserializer; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -47,8 +46,10 @@ public class QFilterCriteria implements Serializable, Cloneable private QCriteriaOperator operator; private List values; - private String otherFieldName; - private AbstractFilterExpression expression; + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // todo - probably implement this as a type of expression - though would require a little special handling i think when evaluating... // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + private String otherFieldName; @@ -101,23 +102,6 @@ public class QFilterCriteria implements Serializable, Cloneable - /******************************************************************************* - ** - *******************************************************************************/ - public QFilterCriteria(String fieldName, QCriteriaOperator operator, AbstractFilterExpression expression) - { - this.fieldName = fieldName; - this.operator = operator; - this.expression = expression; - - /////////////////////////////////////// - // this guy doesn't like to be null? // - /////////////////////////////////////// - this.values = new ArrayList<>(); - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -335,35 +319,4 @@ public class QFilterCriteria implements Serializable, Cloneable return (rs.toString()); } - - - /******************************************************************************* - ** Getter for expression - *******************************************************************************/ - public AbstractFilterExpression getExpression() - { - return (this.expression); - } - - - - /******************************************************************************* - ** Setter for expression - *******************************************************************************/ - public void setExpression(AbstractFilterExpression expression) - { - this.expression = expression; - } - - - - /******************************************************************************* - ** Fluent setter for expression - *******************************************************************************/ - public QFilterCriteria withExpression(AbstractFilterExpression expression) - { - this.expression = expression; - return (this); - } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/AbstractFilterExpression.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/AbstractFilterExpression.java index 723fa818..cc58e9fa 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/AbstractFilterExpression.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/AbstractFilterExpression.java @@ -28,7 +28,7 @@ import java.io.Serializable; /******************************************************************************* ** *******************************************************************************/ -public abstract class AbstractFilterExpression +public abstract class AbstractFilterExpression implements Serializable { /******************************************************************************* ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializer.java index ae4dfd64..ca050b33 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializer.java @@ -23,7 +23,10 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query.serialization; import java.io.IOException; +import java.io.Serializable; import java.util.List; +import java.util.ListIterator; +import java.util.Map; import com.fasterxml.jackson.core.JacksonException; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; @@ -33,6 +36,9 @@ import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; /******************************************************************************* @@ -73,31 +79,39 @@ public class QFilterCriteriaDeserializer extends StdDeserializer values = objectMapper.treeToValue(node.get("values"), List.class); + String fieldName = objectMapper.treeToValue(node.get("fieldName"), String.class); + QCriteriaOperator operator = objectMapper.treeToValue(node.get("operator"), QCriteriaOperator.class); + String otherFieldName = objectMapper.treeToValue(node.get("otherFieldName"), String.class); - AbstractFilterExpression expression = null; - if(node.has("expression")) + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // look at all the values - if any of them are actually meant to be an Expression (instance of subclass of AbstractFilterExpression) // + // they'll have deserialized as a Map, with a "type" key. If that's the case, then re/de serialize them into the proper expression type // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ListIterator valuesIterator = CollectionUtils.nonNullList(values).listIterator(); + while(valuesIterator.hasNext()) { - JsonNode expressionNode = node.get("expression"); - String expressionType = objectMapper.treeToValue(expressionNode.get("type"), String.class); + Object value = valuesIterator.next(); + if(value instanceof Map map && map.containsKey("type")) + { + String expressionType = ValueUtils.getValueAsString(map.get("type")); - //////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // right now, we'll assume that all expression subclasses are in the same package as AbstractFilterExpression // - // so, we can just do a Class.forName on that name, and treeToValue like that. // - // if we ever had to, we could instead switch(expressionType), and do like so... // - // case "NowWithOffset" -> objectMapper.treeToValue(expressionNode, NowWithOffset.class); // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////// - try - { - String className = AbstractFilterExpression.class.getName().replace(AbstractFilterExpression.class.getSimpleName(), expressionType); - expression = (AbstractFilterExpression) objectMapper.treeToValue(expressionNode, Class.forName(className)); - } - catch(Exception e) - { - throw (new IOException("Error deserializing expression of type [" + expressionType + "] inside QFilterCriteria", e)); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // right now, we'll assume that all expression subclasses are in the same package as AbstractFilterExpression // + // so, we can just do a Class.forName on that name, and use JsonUtils.toObject requesting that class. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + try + { + String assumedExpressionJSON = JsonUtils.toJson(map); + + String className = AbstractFilterExpression.class.getName().replace(AbstractFilterExpression.class.getSimpleName(), expressionType); + Serializable replacementValue = (Serializable) JsonUtils.toObject(assumedExpressionJSON, Class.forName(className)); + valuesIterator.set(replacementValue); + } + catch(Exception e) + { + throw (new IOException("Error deserializing criteria value which appeared to be an expression of type [" + expressionType + "] inside QFilterCriteria", e)); + } } } @@ -109,16 +123,7 @@ public class QFilterCriteriaDeserializer extends StdDeserializer T toObject(String json, Class targetClass) throws IOException + { + return (toObject(json, targetClass, null)); + } + + + + /******************************************************************************* + ** De-serialize a json string into an object of the specified class - with + ** customizations on the Jackson ObjectMapper. + **. + ** + ** Internally using jackson - so jackson annotations apply! + ** + *******************************************************************************/ + public static T toObject(String json, Class targetClass, Consumer objectMapperCustomizer) throws IOException { ObjectMapper objectMapper = newObjectMapper(); + if(objectMapperCustomizer != null) + { + objectMapperCustomizer.accept(objectMapper); + } return objectMapper.reader().readValue(json, targetClass); } @@ -172,6 +191,25 @@ public class JsonUtils + /******************************************************************************* + ** De-serialize a json string into an object of the specified class - with + ** customizations on the Jackson ObjectMapper. + ** + ** Internally using jackson - so jackson annotations apply! + ** + *******************************************************************************/ + public static T toObject(String json, TypeReference typeReference, Consumer objectMapperCustomizer) throws IOException + { + ObjectMapper objectMapper = newObjectMapper(); + if(objectMapperCustomizer != null) + { + objectMapperCustomizer.accept(objectMapper); + } + return objectMapper.readValue(json, typeReference); + } + + + /******************************************************************************* ** De-serialize a json string into a JSONObject (string must start with "{") ** diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializerTest.java index 42f7c3da..50f9bc8c 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializerTest.java @@ -30,13 +30,15 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression; import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.Now; import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.ThisOrLastPeriod; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import org.junit.jupiter.api.Test; +import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.BETWEEN; import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.EQUALS; import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.GREATER_THAN; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; /******************************************************************************* @@ -52,6 +54,11 @@ class QFilterCriteriaDeserializerTest extends BaseTest @Test void testDeserialize() throws IOException { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // just put a reference to this class here, so it's a tad easier to find this class via navigation in IDE... // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + new QFilterCriteriaDeserializer(); + { QFilterCriteria criteria = JsonUtils.toObject(""" {"fieldName": "id", "operator": "EQUALS", "values": [1]} @@ -63,14 +70,13 @@ class QFilterCriteriaDeserializerTest extends BaseTest { QFilterCriteria criteria = JsonUtils.toObject(""" - {"fieldName": "createDate", "operator": "GREATER_THAN", "expression": - {"type": "NowWithOffset", "operator": "PLUS", "amount": 5, "timeUnit": "MINUTES"} + {"fieldName": "createDate", "operator": "GREATER_THAN", "values": + [{"type": "NowWithOffset", "operator": "PLUS", "amount": 5, "timeUnit": "MINUTES"}] } """, QFilterCriteria.class); assertEquals("createDate", criteria.getFieldName()); assertEquals(GREATER_THAN, criteria.getOperator()); - assertNull(criteria.getValues()); - AbstractFilterExpression expression = criteria.getExpression(); + AbstractFilterExpression expression = (AbstractFilterExpression) criteria.getValues().get(0); assertThat(expression).isInstanceOf(NowWithOffset.class); NowWithOffset nowWithOffset = (NowWithOffset) expression; assertEquals(5, nowWithOffset.getAmount()); @@ -80,14 +86,31 @@ class QFilterCriteriaDeserializerTest extends BaseTest { QFilterCriteria criteria = JsonUtils.toObject(""" - {"fieldName": "orderDate", "operator": "EQUALS", "expression": {"type": "Now"} } + {"fieldName": "orderDate", "operator": "EQUALS", "values": [{"type": "Now"}] } """, QFilterCriteria.class); assertEquals("orderDate", criteria.getFieldName()); assertEquals(EQUALS, criteria.getOperator()); - assertNull(criteria.getValues()); - AbstractFilterExpression expression = criteria.getExpression(); + AbstractFilterExpression expression = (AbstractFilterExpression) criteria.getValues().get(0); assertThat(expression).isInstanceOf(Now.class); } + { + QFilterCriteria criteria = JsonUtils.toObject(""" + {"fieldName": "orderDate", "operator": "BETWEEN", "values": [{"type": "Now"}, {"type": "ThisOrLastPeriod"}] } + """, QFilterCriteria.class); + assertEquals("orderDate", criteria.getFieldName()); + assertEquals(BETWEEN, criteria.getOperator()); + AbstractFilterExpression expression0 = (AbstractFilterExpression) criteria.getValues().get(0); + assertThat(expression0).isInstanceOf(Now.class); + AbstractFilterExpression expression1 = (AbstractFilterExpression) criteria.getValues().get(1); + assertThat(expression1).isInstanceOf(ThisOrLastPeriod.class); + } + + { + assertThatThrownBy(() -> JsonUtils.toObject(""" + {"fieldName": "orderDate", "operator": "BETWEEN", "values": [{"type": "NotAnExpressionType"}] } + """, QFilterCriteria.class)).hasMessageContaining("Error deserializing criteria value which appeared to be an expression"); + } + } } \ No newline at end of file diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index 54d8ce3c..c0890aff 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -33,6 +33,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.ListIterator; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -57,6 +58,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; @@ -713,14 +715,23 @@ public abstract class AbstractRDBMSAction implements QActionInterface ///////////////////////////////////////////////////////////////////// values = Collections.emptyList(); } - else if(expectedNoOfParams.equals(1) && criterion.getExpression() != null) - { - values = List.of(criterion.getExpression().evaluate()); - } else if(!expectedNoOfParams.equals(values.size())) { throw new IllegalArgumentException("Incorrect number of values given for criteria [" + field.getName() + "]"); } + + ////////////////////////////////////////////////////////////// + // replace any expression-type values with their evaluation // + ////////////////////////////////////////////////////////////// + ListIterator valueListIterator = values.listIterator(); + while(valueListIterator.hasNext()) + { + Serializable value = valueListIterator.next(); + if(value instanceof AbstractFilterExpression expression) + { + valueListIterator.set(expression.evaluate()); + } + } } clauses.add("(" + clause + ")"); diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java index 4e49d82a..b8558e12 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java @@ -29,7 +29,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; @@ -559,7 +558,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest QueryInput queryInput = initQueryRequest(); queryInput.setFilter(new QQueryFilter() .withCriteria(new QFilterCriteria().withFieldName("lastName").withOperator(QCriteriaOperator.EQUALS).withValues(List.of("ExpressionTest"))) - .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withExpression(new Now()))); + .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(new Now())))); QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row"); @@ -569,7 +568,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest QueryInput queryInput = initQueryRequest(); queryInput.setFilter(new QQueryFilter() .withCriteria(new QFilterCriteria().withFieldName("lastName").withOperator(QCriteriaOperator.EQUALS).withValues(List.of("ExpressionTest"))) - .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withExpression(NowWithOffset.plus(2, TimeUnit.DAYS)))); + .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(NowWithOffset.plus(2, ChronoUnit.DAYS))))); QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row"); @@ -579,7 +578,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest QueryInput queryInput = initQueryRequest(); queryInput.setFilter(new QQueryFilter() .withCriteria(new QFilterCriteria().withFieldName("lastName").withOperator(QCriteriaOperator.EQUALS).withValues(List.of("ExpressionTest"))) - .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.GREATER_THAN).withExpression(NowWithOffset.minus(5, TimeUnit.DAYS)))); + .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.GREATER_THAN).withValues(List.of(NowWithOffset.minus(5, ChronoUnit.DAYS))))); QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row"); From 6d6510c22372a048030e81082116f92bf80218a7 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 13 Jul 2023 17:09:18 -0500 Subject: [PATCH 16/24] Add swapMultiLevelMapKeys --- .../backend/core/utils/CollectionUtils.java | 36 +++++++++++++++++++ .../core/utils/CollectionUtilsTest.java | 23 ++++++++++++ 2 files changed, 59 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java index f817b6b3..ecb5aca5 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java @@ -627,4 +627,40 @@ public class CollectionUtils } } + + + /******************************************************************************* + ** Take a multi-level map, e.g., Map{String, Map{Integer, BigDecimal}} + ** and invert it - e.g., to Map{Integer, Map{String, BigDecimal}} + *******************************************************************************/ + public static Map> swapMultiLevelMapKeys(Map> input) + { + if(input == null) + { + return (null); + } + + Map> output = new HashMap<>(); + + for(Map.Entry> entry : input.entrySet()) + { + K1 key1 = entry.getKey(); + Map map1 = entry.getValue(); + + if(map1 != null) + { + for(Map.Entry entry2 : map1.entrySet()) + { + K2 key2 = entry2.getKey(); + V value = entry2.getValue(); + + output.computeIfAbsent(key2, (k) -> new HashMap<>()); + output.get(key2).put(key1, value); + } + } + } + + return (output); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/CollectionUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/CollectionUtilsTest.java index 80425535..62c0573e 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/CollectionUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/CollectionUtilsTest.java @@ -34,6 +34,7 @@ import java.util.TreeMap; import java.util.function.Function; import com.google.gson.reflect.TypeToken; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -594,4 +595,26 @@ class CollectionUtilsTest extends BaseTest } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSwapMultiLevelMapKeys() + { + Map> input = MapBuilder.of( + "A", Map.of(1, "A1", 2, "A2", 3, "A3"), + "B", Map.of(1, "B1", 4, "B4"), + "C", null); + + Map> output = CollectionUtils.swapMultiLevelMapKeys(input); + + assertEquals(MapBuilder.of( + 1, Map.of("A", "A1", "B", "B1"), + 2, Map.of("A", "A2"), + 3, Map.of("A", "A3"), + 4, Map.of("B", "B4")), output); + } + } From 8c3648920d83ad6ebd4effd214afad3bd5c98de2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 13 Jul 2023 17:09:41 -0500 Subject: [PATCH 17/24] Don't audit values for masked field types --- .../qqq/backend/core/actions/audits/DMLAuditAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java index 9ded0791..e9816912 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java @@ -194,7 +194,7 @@ public class DMLAuditAction extends AbstractQActionFunction Date: Thu, 13 Jul 2023 17:10:42 -0500 Subject: [PATCH 18/24] Try to add hints about unrecognized field names (if they're in other api versions) --- .../qqq/api/actions/QRecordApiAdapter.java | 55 ++++++++++++++++++- .../api/actions/QRecordApiAdapterTest.java | 9 ++- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java index 85f0c602..beb902d5 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java @@ -35,6 +35,8 @@ import com.kingsrook.qqq.api.model.APIVersion; import com.kingsrook.qqq.api.model.APIVersionRange; import com.kingsrook.qqq.api.model.actions.ApiFieldCustomValueMapper; import com.kingsrook.qqq.api.model.actions.GetTableApiFieldsInput; +import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaData; +import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaDataContainer; import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData; import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaDataContainer; import com.kingsrook.qqq.api.model.metadata.tables.ApiAssociationMetaData; @@ -289,7 +291,27 @@ public class QRecordApiAdapter if(!unrecognizedFieldNames.isEmpty()) { - throw (new QBadRequestException("Request body contained " + unrecognizedFieldNames.size() + " unrecognized field name" + StringUtils.plural(unrecognizedFieldNames) + ": " + StringUtils.joinWithCommasAndAnd(unrecognizedFieldNames))); + List otherVersionHints = new ArrayList<>(); + try + { + for(String unrecognizedFieldName : unrecognizedFieldNames) + { + String hint = lookForFieldInOtherVersions(unrecognizedFieldName, tableName, apiName, apiVersion); + if(hint != null) + { + otherVersionHints.add(hint); + } + } + } + catch(Exception e) + { + LOG.warn("Error looking for unrecognized field names in other api versions", e); + } + + throw (new QBadRequestException("Request body contained " + + (unrecognizedFieldNames.size() + " unrecognized field name" + StringUtils.plural(unrecognizedFieldNames) + ": " + StringUtils.joinWithCommasAndAnd(unrecognizedFieldNames) + ". ") + + (CollectionUtils.nullSafeIsEmpty(otherVersionHints) ? "" : StringUtils.join(" ", otherVersionHints)) + )); } return (qRecord); @@ -297,6 +319,37 @@ public class QRecordApiAdapter + /******************************************************************************* + ** + *******************************************************************************/ + private static String lookForFieldInOtherVersions(String unrecognizedFieldName, String tableName, String apiName, String apiVersion) throws QException + { + ApiInstanceMetaDataContainer apiInstanceMetaDataContainer = ApiInstanceMetaDataContainer.of(QContext.getQInstance()); + ApiInstanceMetaData apiInstanceMetaData = apiInstanceMetaDataContainer.getApiInstanceMetaData(apiName); + + List versionsWithThisField = new ArrayList<>(); + for(APIVersion supportedVersion : apiInstanceMetaData.getSupportedVersions()) + { + if(!supportedVersion.toString().equals(apiVersion)) + { + Map versionFields = getTableApiFieldMap(new ApiNameVersionAndTableName(apiName, supportedVersion.toString(), tableName)); + if(versionFields.containsKey(unrecognizedFieldName)) + { + versionsWithThisField.add(supportedVersion.toString()); + } + } + } + + if(CollectionUtils.nullSafeHasContents(versionsWithThisField)) + { + return (unrecognizedFieldName + " does not exist in version " + apiVersion + ", but does exist in versions: " + StringUtils.joinWithCommasAndAnd(versionsWithThisField) + ". "); + } + + return (null); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/QRecordApiAdapterTest.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/QRecordApiAdapterTest.java index 5494b9cb..24c6abc0 100644 --- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/QRecordApiAdapterTest.java +++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/QRecordApiAdapterTest.java @@ -158,7 +158,8 @@ class QRecordApiAdapterTest extends BaseTest {"firstName": "Tim", "noOfShoes": 2} """), TestUtils.TABLE_NAME_PERSON, TestUtils.API_NAME, TestUtils.V2022_Q4, true)) .isInstanceOf(QBadRequestException.class) - .hasMessageContaining("unrecognized field name: noOfShoes"); + .hasMessageContaining("unrecognized field name: noOfShoes") + .hasMessageContaining("noOfShoes does not exist in version 2022.Q4, but does exist in versions: 2023.Q1"); ///////////////////////////////////////////////////////////////////////// // current version doesn't have cost field - fail if you send it to us // @@ -167,7 +168,8 @@ class QRecordApiAdapterTest extends BaseTest {"firstName": "Tim", "cost": 2} """), TestUtils.TABLE_NAME_PERSON, TestUtils.API_NAME, TestUtils.V2023_Q1, true)) .isInstanceOf(QBadRequestException.class) - .hasMessageContaining("unrecognized field name: cost"); + .hasMessageContaining("unrecognized field name: cost") + .hasMessageNotContaining("cost does not exist in version 2023.Q1, but does exist in versions: 2023.Q2"); // this field only appears in a future version, not any current/supported versions. ///////////////////////////////// // excluded field always fails // @@ -178,7 +180,8 @@ class QRecordApiAdapterTest extends BaseTest {"firstName": "Tim", "price": 2} """), TestUtils.TABLE_NAME_PERSON, TestUtils.API_NAME, version, true)) .isInstanceOf(QBadRequestException.class) - .hasMessageContaining("unrecognized field name: price"); + .hasMessageContaining("unrecognized field name: price") + .hasMessageNotContaining("price does not exist in version"); // this field never appears, so no message about when it appears. } //////////////////////////////////////////// From 3d2708da23e8c9d84aac8f2730c0672a32a09219 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Jul 2023 14:08:27 -0500 Subject: [PATCH 19/24] CE-535 All more points of overridability, and make keys in existing record map a pair of {fieldName,value} --- .../AbstractTableSyncTransformStep.java | 86 +++++++++++++------ 1 file changed, 62 insertions(+), 24 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java index 6221787d..71769499 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -232,7 +233,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt /////////////////////////////////////////////////////////////////////////////////////////////////// // query to see if we already have those records in the destination (to determine insert/update) // /////////////////////////////////////////////////////////////////////////////////////////////////// - Map existingRecordsByForeignKey = getExistingRecordsByForeignKey(runBackendStepInput, destinationTableForeignKeyField, destinationTableName, sourceKeyList); + Map, QRecord> existingRecordsByForeignKey = getExistingRecordsByForeignKey(runBackendStepInput, destinationTableForeignKeyField, destinationTableName, sourceKeyList); ///////////////////////////////////////////////////////////////// // foreach source record, build the record we'll insert/update // @@ -267,13 +268,10 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt continue; } - ///////////////////////////////////////////////////////////////////////////////////////////////// - // look for the existing record - note - we may need to type-convert here, the sourceKey value // - // from the source table to the destinationKey. e.g., if source table had an integer, and the // - // destination has a string. // - ///////////////////////////////////////////////////////////////////////////////////////////////// - Serializable sourceKeyValueInTargetFieldType = ValueUtils.getValueAsFieldType(destinationForeignKeyField.getType(), sourceKeyValue); - QRecord existingRecord = existingRecordsByForeignKey.get(sourceKeyValueInTargetFieldType); + ////////////////////////////////////////////////////////////// + // look for the existing record, to determine insert/update // + ////////////////////////////////////////////////////////////// + QRecord existingRecord = getExistingRecord(existingRecordsByForeignKey, destinationForeignKeyField, sourceKeyValue); QRecord recordToStore; if(existingRecord != null && config.performUpdates) @@ -333,26 +331,66 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt /******************************************************************************* ** *******************************************************************************/ - protected Map getExistingRecordsByForeignKey(RunBackendStepInput runBackendStepInput, String destinationTableForeignKeyField, String destinationTableName, List sourceKeyList) throws QException + protected QRecord getExistingRecord(Map, QRecord> existingRecordsByForeignKey, QFieldMetaData destinationForeignKeyField, Serializable sourceKeyValue) { - Map existingRecordsByForeignKey = Collections.emptyMap(); - if(!sourceKeyList.isEmpty()) + ////////////////////////////////////////////////////////////////////////////////////////////////// + // note - we may need to type-convert here, the sourceKey value from the source table to // + // the destinationKey. e.g., if source table had an integer, and the destination has a string. // + ////////////////////////////////////////////////////////////////////////////////////////////////// + Serializable sourceKeyValueInTargetFieldType = ValueUtils.getValueAsFieldType(destinationForeignKeyField.getType(), sourceKeyValue); + return (existingRecordsByForeignKey.get(Pair.of(destinationForeignKeyField.getName(), sourceKeyValueInTargetFieldType))); + } + + + + /******************************************************************************* + ** Run the existingRecordQueryFilter - to look in the destinationTable for + ** any records that may need an update (rather than an insert). + ** + ** Generally returns a Map, keyed by a Pair of the destinationTableForeignKeyField + ** and the value in that field. But, for more complex use-cases, one can override + ** the buildExistingRecordsMap method, to make different keys (e.g., if there are + ** two possible destinationTableForeignKeyFields). + *******************************************************************************/ + protected Map, QRecord> getExistingRecordsByForeignKey(RunBackendStepInput runBackendStepInput, String destinationTableForeignKeyField, String destinationTableName, List sourceKeyList) throws QException + { + if(sourceKeyList.isEmpty()) { - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(destinationTableName); - getTransaction().ifPresent(queryInput::setTransaction); - QQueryFilter filter = getExistingRecordQueryFilter(runBackendStepInput, sourceKeyList); - queryInput.setFilter(filter); + return (Collections.emptyMap()); + } - Collection associationNamesToInclude = getAssociationNamesToInclude(); - if(CollectionUtils.nullSafeHasContents(associationNamesToInclude)) - { - queryInput.setIncludeAssociations(true); - queryInput.setAssociationNamesToInclude(associationNamesToInclude); - } + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(destinationTableName); + getTransaction().ifPresent(queryInput::setTransaction); + QQueryFilter filter = getExistingRecordQueryFilter(runBackendStepInput, sourceKeyList); + queryInput.setFilter(filter); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - existingRecordsByForeignKey = CollectionUtils.recordsToMap(queryOutput.getRecords(), destinationTableForeignKeyField); + Collection associationNamesToInclude = getAssociationNamesToInclude(); + if(CollectionUtils.nullSafeHasContents(associationNamesToInclude)) + { + queryInput.setIncludeAssociations(true); + queryInput.setAssociationNamesToInclude(associationNamesToInclude); + } + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + return (buildExistingRecordsMap(destinationTableForeignKeyField, queryOutput.getRecords())); + } + + + + /******************************************************************************* + ** Overridable point where you can, for example, keys in the existingRecordsMap + ** with different fieldNames from the destinationTable. + ** + ** Note, if you're overriding this method, you'll likely also want & need to + ** override getExistingRecord. + *******************************************************************************/ + protected Map, QRecord> buildExistingRecordsMap(String destinationTableForeignKeyField, List existingRecordList) + { + Map, QRecord> existingRecordsByForeignKey = new HashMap<>(); + for(QRecord record : existingRecordList) + { + existingRecordsByForeignKey.put(Pair.of(destinationTableForeignKeyField, record.getValue(destinationTableForeignKeyField)), record); } return (existingRecordsByForeignKey); } From 2db1adc9ab672a6719c4db1982231d86d2925c1d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 17 Jul 2023 11:34:01 -0500 Subject: [PATCH 20/24] CE-536 If records are supplied to the process input, then use them instead of running a query. --- .../etl/streamedwithfrontend/ExtractViaQueryStep.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java index 1f3e776d..637b0f9e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java @@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -80,6 +81,15 @@ public class ExtractViaQueryStep extends AbstractExtractStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { + ///////////////////////////////////////////////////////////////////////// + // if records are already specified in the step input, then use those. // + ///////////////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(runBackendStepInput.getRecords())) + { + getRecordPipe().addRecords(runBackendStepInput.getRecords()); + return; + } + ////////////////////////////////////////////////////////////////// // clone the filter, since we're going to edit it (set a limit) // ////////////////////////////////////////////////////////////////// From 5a5c9a0072697212ed5da154e31aa98247d53516 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 17 Jul 2023 12:16:45 -0500 Subject: [PATCH 21/24] Revert: CE-536 If records are supplied to the process input, then use them instead of running a query. --- .../etl/streamedwithfrontend/ExtractViaQueryStep.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java index 637b0f9e..1f3e776d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java @@ -39,7 +39,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -81,15 +80,6 @@ public class ExtractViaQueryStep extends AbstractExtractStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - ///////////////////////////////////////////////////////////////////////// - // if records are already specified in the step input, then use those. // - ///////////////////////////////////////////////////////////////////////// - if(CollectionUtils.nullSafeHasContents(runBackendStepInput.getRecords())) - { - getRecordPipe().addRecords(runBackendStepInput.getRecords()); - return; - } - ////////////////////////////////////////////////////////////////// // clone the filter, since we're going to edit it (set a limit) // ////////////////////////////////////////////////////////////////// From d62a4c6daf0cf4e381370ac59a43e3d83758f81c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 18 Jul 2023 16:28:32 -0500 Subject: [PATCH 22/24] CE-536 Add getRecordByUniqueKey --- .../processes/utils/RecordLookupHelper.java | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java index 527b7bfc..94ca1986 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java @@ -29,8 +29,11 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -48,8 +51,11 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public class RecordLookupHelper { - private Map> recordMaps = new HashMap<>(); - private Set preloadedKeys = new HashSet<>(); + private Map> recordMaps = new HashMap<>(); + + private Map, QRecord>> uniqueKeyMaps = new HashMap<>(); + + private Set preloadedKeys = new HashSet<>(); private Set> disallowedOneOffLookups = new HashSet<>(); @@ -65,6 +71,25 @@ public class RecordLookupHelper + /******************************************************************************* + ** Fetch a record from a table by a uniqueKey from the table + *******************************************************************************/ + public QRecord getRecordByUniqueKey(String tableName, Map uniqueKey) throws QException + { + String mapKey = tableName + "." + uniqueKey.keySet().stream().sorted().collect(Collectors.joining(",")); + Map, QRecord> recordMap = uniqueKeyMaps.computeIfAbsent(mapKey, (k) -> new HashMap<>()); + + if(!recordMap.containsKey(uniqueKey)) + { + QRecord record = new GetAction().executeForRecord(new GetInput(tableName).withUniqueKey(uniqueKey)); + recordMap.put(uniqueKey, record); + } + + return (recordMap.get(uniqueKey)); + } + + + /******************************************************************************* ** Fetch a record from a table by a key field (doesn't have to be its primary key). *******************************************************************************/ From f5f2cc50074021ce43c10a9ee638a581573919b5 Mon Sep 17 00:00:00 2001 From: t-samples Date: Thu, 20 Jul 2023 09:32:52 -0500 Subject: [PATCH 23/24] CE-508 - Updated to support setCredentialsInHeader --- .../module/api/actions/BaseAPIActionUtil.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 3856bd6b..7314e53e 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -73,6 +73,7 @@ import com.kingsrook.qqq.backend.module.api.model.OutboundAPILog; import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendMetaData; import com.kingsrook.qqq.backend.module.api.model.metadata.APITableBackendDetails; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.BooleanUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpResponse; @@ -728,19 +729,28 @@ public class BaseAPIActionUtil // this is not generally meant to be put in the meta data by the app programmer - rather, we're just using // // it as a "cheap & easy" way to "cache" the token within our process's memory... // //////////////////////////////////////////////////////////////////////////////////////////////////////////////// - String accessToken = ValueUtils.getValueAsString(backendMetaData.getCustomValue("accessToken")); + String accessToken = ValueUtils.getValueAsString(backendMetaData.getCustomValue("accessToken")); + Boolean setCredentialsInHeader = BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(backendMetaData.getCustomValue("setCredentialsInHeader"))); if(!StringUtils.hasContent(accessToken)) { String fullURL = backendMetaData.getBaseUrl() + "oauth/token"; - String postBody = "grant_type=client_credentials&client_id=" + backendMetaData.getClientId() + "&client_secret=" + backendMetaData.getClientSecret(); + String postBody = "grant_type=client_credentials"; - LOG.info("Fetching OAuth2 token from " + fullURL); + if(!setCredentialsInHeader) + { + postBody += "&client_id=" + backendMetaData.getClientId() + "&client_secret=" + backendMetaData.getClientSecret(); + } try(CloseableHttpClient client = HttpClients.custom().setConnectionManager(new PoolingHttpClientConnectionManager()).build()) { HttpPost request = new HttpPost(fullURL); request.setEntity(new StringEntity(postBody)); + + if(setCredentialsInHeader) + { + request.addHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getClientId(), backendMetaData.getClientSecret())); + } request.addHeader("Content-Type", "application/x-www-form-urlencoded; charset=utf-8"); HttpResponse response = executeOAuthTokenRequest(client, request); From bff8a0f78aecb1cac6e0c4f6e172a67349c2612d Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Mon, 24 Jul 2023 15:16:37 -0500 Subject: [PATCH 24/24] Update versions for release --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 20fde93f..0303dfe5 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ - 0.16.0-SNAPSHOT + 0.16.0 UTF-8 UTF-8