From 8dbf7fe4cdf3ff218d0c11e5c194d2a578517a3d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 5 Jul 2024 12:57:07 -0500 Subject: [PATCH 01/19] CE-1460 Construct a new, clean QueryJoin object for the second Aggregate call (as JoinsContext changes the one it takes in during the first call, leading to different join conditions being in place, causing second query to potentially fail) --- .../columnstats/ColumnStatsStep.java | 77 ++++++++++++------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java index 21584e8e..cbe1e1a9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java @@ -136,29 +136,11 @@ public class ColumnStatsStep implements BackendStep filter = new QQueryFilter(); } - QueryJoin queryJoin = null; - QTableMetaData table = QContext.getQInstance().getTable(tableName); - QFieldMetaData field = null; - if(fieldName.contains(".")) - { - String[] parts = fieldName.split("\\.", 2); - for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) - { - if(exposedJoin.getJoinTable().equals(parts[0])) - { - field = QContext.getQInstance().getTable(exposedJoin.getJoinTable()).getField(parts[1]); - queryJoin = new QueryJoin() - .withJoinTable(exposedJoin.getJoinTable()) - .withSelect(true) - .withType(QueryJoin.Type.INNER); - break; - } - } - } - else - { - field = table.getField(fieldName); - } + QTableMetaData table = QContext.getQInstance().getTable(tableName); + + FieldAndQueryJoin fieldAndQueryJoin = getFieldAndQueryJoin(table, fieldName); + QFieldMetaData field = fieldAndQueryJoin.field(); + QueryJoin queryJoin = fieldAndQueryJoin.queryJoin(); if(field == null) { @@ -213,7 +195,7 @@ public class ColumnStatsStep implements BackendStep filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)); filter.withOrderBy(new QFilterOrderByGroupBy(groupBy)); - Integer limit = 1000; // too big? + Integer limit = 1000; AggregateInput aggregateInput = new AggregateInput(); aggregateInput.withAggregate(aggregate); aggregateInput.withGroupBy(groupBy); @@ -223,7 +205,11 @@ public class ColumnStatsStep implements BackendStep if(queryJoin != null) { - aggregateInput.withQueryJoin(queryJoin); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // re-construct this queryJoin object - just because, the JoinsContext edits the previous one, so we can make some failing-joins otherwise... // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + fieldAndQueryJoin = getFieldAndQueryJoin(table, fieldName); + aggregateInput.withQueryJoin(fieldAndQueryJoin.queryJoin()); } AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput); @@ -238,7 +224,7 @@ public class ColumnStatsStep implements BackendStep value = Instant.parse(value + ":00:00Z"); } - Integer count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate)); + Integer count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate)); valueCounts.add(new QRecord().withValue(fieldName, value).withValue("count", count)); } @@ -526,4 +512,43 @@ public class ColumnStatsStep implements BackendStep return (null); } + + + /******************************************************************************* + ** + *******************************************************************************/ + private FieldAndQueryJoin getFieldAndQueryJoin(QTableMetaData table, String fieldName) + { + QFieldMetaData field = null; + QueryJoin queryJoin = null; + if(fieldName.contains(".")) + { + String[] parts = fieldName.split("\\.", 2); + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + if(exposedJoin.getJoinTable().equals(parts[0])) + { + field = QContext.getQInstance().getTable(exposedJoin.getJoinTable()).getField(parts[1]); + queryJoin = new QueryJoin() + .withJoinTable(exposedJoin.getJoinTable()) + .withSelect(true) + .withType(QueryJoin.Type.INNER); + break; + } + } + } + else + { + field = table.getField(fieldName); + } + + return (new FieldAndQueryJoin(field, queryJoin)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private record FieldAndQueryJoin(QFieldMetaData field, QueryJoin queryJoin) {} } From 172b25f33e06bb8a71bf0537749a6fe208b5b784 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 09:49:43 -0500 Subject: [PATCH 02/19] CE-1460 Fix in makeFromClause, to flip join before getting names out of it. Fixes a case where the JoinContext can send a backward join this far. --- .../rdbms/actions/AbstractRDBMSAction.java | 55 +++++-- .../actions/RDBMSQueryActionJoinsTest.java | 153 +++++++++++++++++- 2 files changed, 188 insertions(+), 20 deletions(-) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index 082759bc..f9e12361 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -245,31 +245,64 @@ public abstract class AbstractRDBMSAction *******************************************************************************/ protected String makeFromClause(QInstance instance, String tableName, JoinsContext joinsContext, List params) { + ////////////////////////////////////////////////////////////////////// + // start with the main table - un-aliased (well, aliased as itself) // + ////////////////////////////////////////////////////////////////////// StringBuilder rs = new StringBuilder(escapeIdentifier(getTableName(instance.getTable(tableName))) + " AS " + escapeIdentifier(tableName)); + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // sort the query joins from the main table "outward"... // + // this might not be perfect, e.g., for cases where what we actually might need is a tree of joins... // + //////////////////////////////////////////////////////////////////////////////////////////////////////// List queryJoins = sortQueryJoinsForFromClause(tableName, joinsContext.getQueryJoins()); + + //////////////////////////////////////////////////////// + // iterate over joins, adding to the from clause (rs) // + //////////////////////////////////////////////////////// for(QueryJoin queryJoin : queryJoins) { - QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable()); - String tableNameOrAlias = queryJoin.getJoinTableOrItsAlias(); + QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable()); + String joinTableNameOrAlias = queryJoin.getJoinTableOrItsAlias(); + //////////////////////////////////////////////////////// + // add the ` JOIN table AS alias` bit to the rs // + //////////////////////////////////////////////////////// rs.append(" ").append(queryJoin.getType()).append(" JOIN ") .append(escapeIdentifier(getTableName(joinTable))) - .append(" AS ").append(escapeIdentifier(tableNameOrAlias)); + .append(" AS ").append(escapeIdentifier(joinTableNameOrAlias)); - //////////////////////////////////////////////////////////// - // find the join in the instance, to set the 'on' clause // - //////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + // find the join in the instance, for building the ON clause // + // append each sub-clause (condition) into a list, for later joining with AND // + //////////////////////////////////////////////////////////////////////////////// List joinClauseList = new ArrayList<>(); String baseTableName = Objects.requireNonNullElse(joinsContext.resolveTableNameOrAliasToTableName(queryJoin.getBaseTableOrAlias()), tableName); QJoinMetaData joinMetaData = Objects.requireNonNull(queryJoin.getJoinMetaData(), () -> "Could not find a join between tables [" + baseTableName + "][" + queryJoin.getJoinTable() + "]"); + ////////////////////////////////////////////////// + // loop over join-ons (e.g., multi-column join) // + ////////////////////////////////////////////////// for(JoinOn joinOn : joinMetaData.getJoinOns()) { + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // figure out if the join needs flipped. We want its left table to equal the queryJoin's base table. // + //////////////////////////////////////////////////////////////////////////////////////////////////////// QTableMetaData leftTable = instance.getTable(joinMetaData.getLeftTable()); QTableMetaData rightTable = instance.getTable(joinMetaData.getRightTable()); + if(!joinMetaData.getLeftTable().equals(baseTableName)) + { + joinOn = joinOn.flip(); + QTableMetaData tmpTable = leftTable; + leftTable = rightTable; + rightTable = tmpTable; + } + + //////////////////////////////////////////////////////////// + // get the table-names-or-aliases to use in the ON clause // + //////////////////////////////////////////////////////////// String baseTableOrAlias = queryJoin.getBaseTableOrAlias(); + String joinTableOrAlias = queryJoin.getJoinTableOrItsAlias(); if(baseTableOrAlias == null) { baseTableOrAlias = leftTable.getName(); @@ -279,15 +312,6 @@ public abstract class AbstractRDBMSAction } } - String joinTableOrAlias = queryJoin.getJoinTableOrItsAlias(); - if(!joinMetaData.getLeftTable().equals(baseTableName)) - { - joinOn = joinOn.flip(); - QTableMetaData tmpTable = leftTable; - leftTable = rightTable; - rightTable = tmpTable; - } - joinClauseList.add(escapeIdentifier(baseTableOrAlias) + "." + escapeIdentifier(getColumnName(leftTable.getField(joinOn.getLeftField()))) + " = " + escapeIdentifier(joinTableOrAlias) @@ -938,6 +962,7 @@ public abstract class AbstractRDBMSAction { try { + params = Objects.requireNonNullElse(params, Collections.emptyList()); params = params.size() <= 100 ? params : params.subList(0, 99); ///////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java index 12993de3..22480177 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java @@ -26,11 +26,14 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; @@ -47,6 +50,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; +import com.kingsrook.qqq.backend.core.utils.collections.SetBuilder; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -54,6 +58,7 @@ import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; /******************************************************************************* @@ -69,10 +74,6 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest public void beforeEach() throws Exception { super.primeTestDatabase(); - - AbstractRDBMSAction.setLogSQL(true); - AbstractRDBMSAction.setLogSQLReformat(true); - AbstractRDBMSAction.setLogSQLOutput("system.out"); } @@ -909,7 +910,7 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest ** *******************************************************************************/ @Test - void testMultipleReversedDirectionJoinsBetweenSameTables() throws QException + void testMultipleReversedDirectionJoinsBetweenSameTablesAllAccessKey() throws QException { QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); @@ -952,6 +953,76 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testMultipleReversedDirectionJoinsBetweenSameTables() throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////////// + // primer sql file loads 2 instructions for order 1, 3 for order 2, then 1 for all the others. // + // let's make things a bit more interesting by deleting the 2 for order 1 // + ///////////////////////////////////////////////////////////////////////////////////////////////// + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + DeleteInput deleteInput = new DeleteInput(); + deleteInput.setTableName(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS); + deleteInput.setQueryFilter(new QQueryFilter(new QFilterCriteria("orderId", QCriteriaOperator.EQUALS, 1))); + new DeleteAction().execute(deleteInput); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + + // Integer noOfOrders = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER)).getCount(); + // Integer noOfOrderInstructions = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS)).getCount(); + + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure we can join on order.current_order_instruction_id = order_instruction.id -- and that we get back 1 row per order // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS).withSelect(true).withJoinMetaData(QContext.getQInstance().getJoin("orderJoinCurrentOrderInstructions"))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size()); + } + + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure we can join on order.current_order_instruction_id = order_instruction.id -- and that we get back 1 row per order // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS).withSelect(true).withType(QueryJoin.Type.LEFT).withJoinMetaData(QContext.getQInstance().getJoin("orderJoinCurrentOrderInstructions"))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size()); + } + + /* + { + //////////////////////////////////////////////////////////////////////////////////////////////// + // assert that the query succeeds (based on exposed join) if the joinMetaData isn't specified // + //////////////////////////////////////////////////////////////////////////////////////////////// + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS)); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + // assertEquals(noOfOrders, queryOutput.getRecords().size()); + } + + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure we can join on order.id = order_instruction.order_id (e.g., not the exposed one used above) -- and that we get back 1 row per order instruction // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS).withJoinMetaData(QContext.getQInstance().getJoin("orderInstructionsJoinOrder"))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + // assertEquals(noOfOrderInstructions, queryOutput.getRecords().size()); + } + */ + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -992,6 +1063,32 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest + /******************************************************************************* + ** We had, at one time, a bug where, for tables with 2 joins between each other, + ** an ON clause could get written using the wrong table name in one part. + ** + ** With that bug, this QueryAction.execute would throw an SQL Exception. + ** + ** So this test, just makes sure that no such exception gets thrown. + *******************************************************************************/ + @Test + void testFlippedJoinForOnClause() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER)); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertFalse(queryOutput.getRecords().isEmpty()); + + //////////////////////////////////// + // if no exception, then we pass. // + //////////////////////////////////// + } + + + /******************************************************************************* ** Addressing a regression where a table was brought into a query for its ** security field, but it was a write-scope lock, so, it shouldn't have been. @@ -1029,4 +1126,50 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest assertEquals(5, new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON)).getRecords().size()); } + + + /******************************************************************************* + ** scenario: + ** order LEFT JOIN shipment. both have storeId as security field. + ** not all orders have a shipment (but due to left-join, expect to get all + ** orders back, even if missing a shipment). + ** + ** If the security clause for shipment is in the WHERE clause, it will "negate" + ** the effect of the LEFT JOIN (unless it includes, e.g., "OR id IS NULL"). + ** + ** At one point, we tried AND'ing such a security clause to the JOIN ... ON, + ** but that is frequently causing issues... + *******************************************************************************/ + @Test + void testLeftJoinSecurityClause() throws QException + { + ////////////////////////////////////////////////////////////////////////////////// + // scenario: // + // order LEFT JOIN shipment. both have storeId as security field. // + // not all orders have a shipment (but due to left-join, expect to get all // + // orders back, even if missing a shipment). // + // // + // If the security clause for shipment is in the WHERE clause, it will "negate" // + // the effect of the LEFT JOIN (unless it includes, e.g., "OR id IS NULL"). // + // // + // At one point, we tried AND'ing such a security clause to the JOIN ... ON, // + // but that is frequently causing issues... // + // // + // order table has 3 rows for store 1: // + // - id=1 has 1 shipment, but assigned to the wrong store! // + // - id=2 has no shipments. // + // - id=3 has 2 shipments. // + ////////////////////////////////////////////////////////////////////////////////// + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHIPMENT).withSelect(true).withType(QueryJoin.Type.LEFT)); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + List records = queryOutput.getRecords(); + assertEquals(4, records.size(), "expected no of records"); + assertEquals(SetBuilder.of(1), records.stream().map(r -> r.getValue("storeId")).collect(Collectors.toSet())); + assertEquals(SetBuilder.of(null, 1), records.stream().map(r -> r.getValue(TestUtils.TABLE_NAME_SHIPMENT + ".storeId")).collect(Collectors.toSet())); + } + } From 6b7fb21d760da1b751862c296b469ff4ea8c06bc Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 09:49:59 -0500 Subject: [PATCH 03/19] CE-1460 Initial checkin --- .../ColumnStatsFullInstanceVerifier.java | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifier.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifier.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifier.java new file mode 100644 index 00000000..8e21c3a6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifier.java @@ -0,0 +1,126 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.columnstats; + + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Utility for verifying that the ColumnStats process works for all fields, + ** on all tables, and all exposed joins. + ** + ** Meant for use within a unit test, or maybe as part of an instance's boot-up/ + ** validation. + *******************************************************************************/ +public class ColumnStatsFullInstanceVerifier +{ + private static final QLogger LOG = QLogger.getLogger(ColumnStatsFullInstanceVerifier.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void verify(Collection tables) throws QException + { + Map, Exception> caughtExceptions = new LinkedHashMap<>(); + for(QTableMetaData table : tables) + { + if(table.isCapabilityEnabled(QContext.getQInstance().getBackendForTable(table.getName()), Capability.QUERY_STATS)) + { + LOG.info("Verifying ColumnStats on table", logPair("tableName", table.getName())); + for(QFieldMetaData field : table.getFields().values()) + { + runColumnStats(table.getName(), field.getName(), caughtExceptions); + } + + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + QTableMetaData joinTable = QContext.getQInstance().getTable(exposedJoin.getJoinTable()); + for(QFieldMetaData field : joinTable.getFields().values()) + { + runColumnStats(table.getName(), joinTable.getName() + "." + field.getName(), caughtExceptions); + } + } + } + } + + // log out an exceptions caught + if(!caughtExceptions.isEmpty()) + { + for(Map.Entry, Exception> entry : caughtExceptions.entrySet()) + { + LOG.info("Caught an exception verifying column stats", entry.getValue(), logPair("tableName", entry.getKey().getA()), logPair("fieldName", entry.getKey().getB())); + } + throw (new QException("Column Status Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size()))); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void runColumnStats(String tableName, String fieldName, Map, Exception> caughtExceptions) throws QException + { + try + { + RunBackendStepInput input = new RunBackendStepInput(); + input.addValue("tableName", tableName); + input.addValue("fieldName", fieldName); + RunBackendStepOutput output = new RunBackendStepOutput(); + new ColumnStatsStep().run(input, output); + } + catch(QException e) + { + Throwable rootException = ExceptionUtils.getRootException(e); + if(rootException instanceof QException && rootException.getMessage().contains("not supported for this field's data type")) + { + //////////////////////////////////////////////// + // ignore this exception, it's kinda expected // + //////////////////////////////////////////////// + LOG.debug("Caught an expected-exception in column stats", e, logPair("tableName", tableName), logPair("fieldName", fieldName)); + } + else + { + caughtExceptions.put(Pair.of(tableName, fieldName), e); + } + } + } +} From 385f4c20e5d90ed1523df5b345504097cfb840e5 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 10:19:34 -0500 Subject: [PATCH 04/19] Add overload of executeStatement, that takes the SQL string, for including in an explicit LOG.warn upon SQLException. Add similar catch(SQLException) { LOG; throw } blocks to other execute methods. --- .../rdbms/actions/RDBMSAggregateAction.java | 2 +- .../rdbms/actions/RDBMSCountAction.java | 2 +- .../rdbms/actions/RDBMSQueryAction.java | 2 +- .../module/rdbms/jdbc/QueryManager.java | 73 +++++++++++++++---- 4 files changed, 62 insertions(+), 17 deletions(-) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java index 21a2052f..0f4e7400 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java @@ -116,7 +116,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega actionTimeoutHelper = new ActionTimeoutHelper(aggregateInput.getTimeoutSeconds(), TimeUnit.SECONDS, new StatementTimeoutCanceller(statement, sql)); actionTimeoutHelper.start(); - QueryManager.executeStatement(statement, ((ResultSet resultSet) -> + QueryManager.executeStatement(statement, sql, ((ResultSet resultSet) -> { ///////////////////////////////////////////////////////////////////////// // once we've started getting results, go ahead and cancel the timeout // diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java index ba167674..f2e40953 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java @@ -96,7 +96,7 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf actionTimeoutHelper = new ActionTimeoutHelper(countInput.getTimeoutSeconds(), TimeUnit.SECONDS, new StatementTimeoutCanceller(statement, sql)); actionTimeoutHelper.start(); - QueryManager.executeStatement(statement, ((ResultSet resultSet) -> + QueryManager.executeStatement(statement, sql, ((ResultSet resultSet) -> { ///////////////////////////////////////////////////////////////////////// // once we've started getting results, go ahead and cancel the timeout // diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index f39ab0f2..fc178fed 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -170,7 +170,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf ////////////////////////////////////////////// QueryOutput queryOutput = new QueryOutput(queryInput); - QueryManager.executeStatement(statement, ((ResultSet resultSet) -> + QueryManager.executeStatement(statement, sql, ((ResultSet resultSet) -> { ///////////////////////////////////////////////////////////////////////// // once we've started getting results, go ahead and cancel the timeout // diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java index 94d81dfd..f1e31ffe 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java @@ -32,6 +32,7 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; +import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; import java.time.Instant; @@ -56,6 +57,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValu import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import org.apache.commons.lang.NotImplementedException; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -121,6 +123,17 @@ public class QueryManager ** customized settings/optimizations). *******************************************************************************/ public static void executeStatement(PreparedStatement statement, ResultSetProcessor processor, Object... params) throws SQLException, QException + { + executeStatement(statement, null, processor, params); + } + + + + /******************************************************************************* + ** Let the caller provide their own prepared statement (e.g., possibly with some + ** customized settings/optimizations). + *******************************************************************************/ + public static void executeStatement(PreparedStatement statement, CharSequence sql, ResultSetProcessor processor, Object... params) throws SQLException, QException { ResultSet resultSet = null; @@ -136,6 +149,14 @@ public class QueryManager processor.processResultSet(resultSet); } } + catch(SQLException e) + { + if(sql != null) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + } + throw (e); + } finally { if(resultSet != null) @@ -372,10 +393,17 @@ public class QueryManager *******************************************************************************/ public static PreparedStatement executeUpdate(Connection connection, String sql, Object... params) throws SQLException { - PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); - incrementStatistic(STAT_QUERIES_RAN); - statement.executeUpdate(); - return (statement); + try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) + { + incrementStatistic(STAT_QUERIES_RAN); + statement.executeUpdate(); + return (statement); + } + catch(SQLException e) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + throw (e); + } } @@ -385,10 +413,17 @@ public class QueryManager *******************************************************************************/ public static PreparedStatement executeUpdate(Connection connection, String sql, List params) throws SQLException { - PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); - incrementStatistic(STAT_QUERIES_RAN); - statement.executeUpdate(); - return (statement); + try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) + { + incrementStatistic(STAT_QUERIES_RAN); + statement.executeUpdate(); + return (statement); + } + catch(SQLException e) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + throw (e); + } } @@ -436,6 +471,11 @@ public class QueryManager statement.executeUpdate(); return (statement.getUpdateCount()); } + catch(SQLException e) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + throw (e); + } } @@ -488,19 +528,24 @@ public class QueryManager *******************************************************************************/ public static List executeInsertForGeneratedIds(Connection connection, String sql, List params) throws SQLException { - List rs = new ArrayList<>(); try(PreparedStatement statement = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS)) { bindParams(params.toArray(), statement); incrementStatistic(STAT_QUERIES_RAN); statement.executeUpdate(); - ResultSet generatedKeys = statement.getGeneratedKeys(); + ResultSet generatedKeys = statement.getGeneratedKeys(); + List rs = new ArrayList<>(); while(generatedKeys.next()) { rs.add(getInteger(generatedKeys, 1)); } + return (rs); + } + catch(SQLException e) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + throw (e); } - return (rs); } @@ -747,14 +792,14 @@ public class QueryManager else if(value instanceof LocalDate ld) { @SuppressWarnings("deprecation") - java.sql.Date date = new java.sql.Date(ld.getYear() - 1900, ld.getMonthValue() - 1, ld.getDayOfMonth()); + Date date = new Date(ld.getYear() - 1900, ld.getMonthValue() - 1, ld.getDayOfMonth()); statement.setDate(index, date); return (1); } else if(value instanceof LocalTime lt) { @SuppressWarnings("deprecation") - java.sql.Time time = new java.sql.Time(lt.getHour(), lt.getMinute(), lt.getSecond()); + Time time = new Time(lt.getHour(), lt.getMinute(), lt.getSecond()); statement.setTime(index, time); return (1); } @@ -943,7 +988,7 @@ public class QueryManager } else { - statement.setDate(index, new java.sql.Date(value.getTime())); + statement.setDate(index, new Date(value.getTime())); } } From bce9af06fbb80012be1a5b57f24a625e4000d425 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 10:20:45 -0500 Subject: [PATCH 05/19] Move logSQL calls into finally blocks, to happen upon success or exception. --- .../rdbms/actions/RDBMSAggregateAction.java | 6 +++-- .../rdbms/actions/RDBMSCountAction.java | 8 +++--- .../rdbms/actions/RDBMSDeleteAction.java | 24 +++++++++++------ .../rdbms/actions/RDBMSInsertAction.java | 17 +++++++----- .../rdbms/actions/RDBMSQueryAction.java | 9 ++----- .../rdbms/actions/RDBMSUpdateAction.java | 26 +++++++++++++------ 6 files changed, 56 insertions(+), 34 deletions(-) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java index 0f4e7400..b174373b 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java @@ -168,8 +168,10 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega }), params); } - - logSQL(sql, params, mark); + finally + { + logSQL(sql, params, mark); + } return rs; } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java index f2e40953..a24890f1 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java @@ -84,10 +84,10 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf setSqlAndJoinsInQueryStat(sql, joinsContext); CountOutput rs = new CountOutput(); + long mark = System.currentTimeMillis(); + try(Connection connection = getConnection(countInput)) { - long mark = System.currentTimeMillis(); - statement = connection.prepareStatement(sql); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -116,7 +116,9 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf setQueryStatFirstResultTime(); }), params); - + } + finally + { logSQL(sql, params, mark); } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java index dd9cf209..5d4cddf8 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java @@ -212,13 +212,16 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte // LOG.debug("rowCount 0 trying to delete [" + tableName + "][" + primaryKey + "]"); // deleteOutput.addRecordWithError(new QRecord(table, primaryKey).withError("Record was not deleted (but no error was given from the database)")); // } - logSQL(sql, List.of(primaryKey), mark); } catch(Exception e) { LOG.debug("Exception trying to delete [" + tableName + "][" + primaryKey + "]", e); deleteOutput.addRecordWithError(new QRecord(table, primaryKey).withError(new SystemErrorStatusMessage("Record was not deleted: " + e.getMessage()))); } + finally + { + logSQL(sql, List.of(primaryKey), mark); + } } @@ -228,13 +231,14 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte *******************************************************************************/ public void doDeleteList(Connection connection, QTableMetaData table, List primaryKeys, DeleteOutput deleteOutput) throws QException { + long mark = System.currentTimeMillis(); + String sql = null; + try { - long mark = System.currentTimeMillis(); - String tableName = getTableName(table); String primaryKeyName = getColumnName(table.getField(table.getPrimaryKeyField())); - String sql = "DELETE FROM " + sql = "DELETE FROM " + escapeIdentifier(tableName) + " WHERE " + escapeIdentifier(primaryKeyName) @@ -246,13 +250,15 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte Integer rowCount = QueryManager.executeUpdateForRowCount(connection, sql, primaryKeys); deleteOutput.addToDeletedRecordCount(rowCount); - - logSQL(sql, primaryKeys, mark); } catch(Exception e) { throw new QException("Error executing delete: " + e.getMessage(), e); } + finally + { + logSQL(sql, primaryKeys, mark); + } } @@ -282,12 +288,14 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte { int rowCount = QueryManager.executeUpdateForRowCount(connection, sql, params); deleteOutput.setDeletedRecordCount(rowCount); - - logSQL(sql, params, mark); } catch(Exception e) { throw new QException("Error executing delete with filter: " + e.getMessage(), e); } + finally + { + logSQL(sql, params, mark); + } } } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java index 101d96f4..b58d1386 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java @@ -57,9 +57,13 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte InsertOutput rs = new InsertOutput(); QTableMetaData table = insertInput.getTable(); - Connection connection = null; + Connection connection = null; boolean needToCloseConnection = false; + StringBuilder sql = null; + List params = null; + Long mark = null; + try { List insertableFields = table.getFields().values().stream() @@ -88,10 +92,10 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte for(List page : CollectionUtils.getPages(insertInput.getRecords(), QueryManager.PAGE_SIZE)) { - String tableName = escapeIdentifier(getTableName(table)); - StringBuilder sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(columns).append(") VALUES"); - List params = new ArrayList<>(); - int recordIndex = 0; + String tableName = escapeIdentifier(getTableName(table)); + sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(columns).append(") VALUES"); + params = new ArrayList<>(); + int recordIndex = 0; ////////////////////////////////////////////////////// // for each record in the page: // @@ -133,7 +137,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte continue; } - Long mark = System.currentTimeMillis(); + mark = System.currentTimeMillis(); /////////////////////////////////////////////////////////// // execute the insert, then foreach record in the input, // @@ -163,6 +167,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte } catch(Exception e) { + logSQL(sql, params, mark); throw new QException("Error executing insert: " + e.getMessage(), e); } finally diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index fc178fed..2f3ab9e0 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -223,17 +223,12 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf }), params); - logSQL(sql, params, mark); - return queryOutput; } - catch(Exception e) - { - logSQL(sql, params, mark); - throw (e); - } finally { + logSQL(sql, params, mark); + if(actionTimeoutHelper != null) { ///////////////////////////////////////// diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java index 54276782..627b9c60 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java @@ -179,10 +179,15 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte //////////////////////////////////////////////////////////////////////////////// // let query manager do the batch updates - note that it will internally page // //////////////////////////////////////////////////////////////////////////////// - QueryManager.executeBatchUpdate(connection, sql, values); - incrementStatus(updateInput, recordList.size()); - - logSQL(sql, values, mark); + try + { + QueryManager.executeBatchUpdate(connection, sql, values); + incrementStatus(updateInput, recordList.size()); + } + finally + { + logSQL(sql, values, mark); + } } @@ -249,10 +254,15 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte ///////////////////////////////////// // let query manager do the update // ///////////////////////////////////// - QueryManager.executeUpdate(connection, sql, params); - incrementStatus(updateInput, page.size()); - - logSQL(sql, params, mark); + try + { + QueryManager.executeUpdate(connection, sql, params); + incrementStatus(updateInput, page.size()); + } + finally + { + logSQL(sql, params, mark); + } } } From 0d2e6012a3631208ade57f9e517b5e51eef9092a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 10:21:59 -0500 Subject: [PATCH 06/19] Remove "aurora" as literal value for rdbmsBackend vendor (in favor of VENDOR_AURORA_MYSQL constant) --- .../backend/module/rdbms/actions/RDBMSQueryAction.java | 5 +---- .../backend/module/rdbms/jdbc/ConnectionManager.java | 10 ++-------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 2f3ab9e0..5b687cd7 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -361,10 +361,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf { RDBMSBackendMetaData rdbmsBackendMetaData = (RDBMSBackendMetaData) queryInput.getBackend(); - //////////////////////////////////////////////////////////////////////////// - // todo - remove "aurora" - it's a legacy value here for a staged rollout // - //////////////////////////////////////////////////////////////////////////// - if(RDBMSBackendMetaData.VENDOR_MYSQL.equals(rdbmsBackendMetaData.getVendor()) || RDBMSBackendMetaData.VENDOR_AURORA_MYSQL.equals(rdbmsBackendMetaData.getVendor()) || "aurora".equals(rdbmsBackendMetaData.getVendor())) + if(RDBMSBackendMetaData.VENDOR_MYSQL.equals(rdbmsBackendMetaData.getVendor()) || RDBMSBackendMetaData.VENDOR_AURORA_MYSQL.equals(rdbmsBackendMetaData.getVendor())) { ////////////////////////////////////////////////////////////////////////////////////////////////////// // mysql "optimization", presumably here - from Result Set section of // diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java index 084e5df1..5ec62e34 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java @@ -150,10 +150,7 @@ public class ConnectionManager return switch(backend.getVendor()) { - //////////////////////////////////////////////////////////////////////////// - // todo - remove "aurora" - it's a legacy value here for a staged rollout // - //////////////////////////////////////////////////////////////////////////// - case RDBMSBackendMetaData.VENDOR_MYSQL, RDBMSBackendMetaData.VENDOR_AURORA_MYSQL, "aurora" -> "com.mysql.cj.jdbc.Driver"; + case RDBMSBackendMetaData.VENDOR_MYSQL, RDBMSBackendMetaData.VENDOR_AURORA_MYSQL -> "com.mysql.cj.jdbc.Driver"; case RDBMSBackendMetaData.VENDOR_H2 -> "org.h2.Driver"; default -> throw (new IllegalStateException("We do not know what jdbc driver to use for vendor name [" + backend.getVendor() + "]. Try setting jdbcDriverClassName in your backend meta data.")); }; @@ -178,10 +175,7 @@ public class ConnectionManager //////////////////////////////////////////////////////////////// // jdbcURL = "jdbc:mysql:aws://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=CONVERT_TO_NULL"; - //////////////////////////////////////////////////////////////////////////// - // todo - remove "aurora" - it's a legacy value here for a staged rollout // - //////////////////////////////////////////////////////////////////////////// - case RDBMSBackendMetaData.VENDOR_AURORA_MYSQL, "aurora" -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false"; + case RDBMSBackendMetaData.VENDOR_AURORA_MYSQL -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false"; case RDBMSBackendMetaData.VENDOR_MYSQL -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull"; case RDBMSBackendMetaData.VENDOR_H2 -> "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1"; default -> throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor()); From 7f23a0da79f25e6a5386de5d8d2ea892237f18a2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 10:22:50 -0500 Subject: [PATCH 07/19] Add LOG.info plus explicit QPermissionDeniedException for null inputs to various checkXPermissionThrowing methods (instead of null pointers) --- .../permissions/PermissionsHelper.java | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java index ab1b56ca..82af5ed7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java @@ -49,6 +49,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -78,6 +79,12 @@ public class PermissionsHelper warnAboutPermissionSubTypeForTables(permissionSubType); QTableMetaData table = QContext.getQInstance().getTable(tableName); + if(table == null) + { + LOG.info("Throwing a permission denied exception in response to a non-existent table name", logPair("tableName", tableName)); + throw (new QPermissionDeniedException("Permission denied.")); + } + commonCheckPermissionThrowing(getEffectivePermissionRules(table, QContext.getQInstance()), permissionSubType, table.getName()); } @@ -184,7 +191,14 @@ public class PermissionsHelper *******************************************************************************/ public static void checkProcessPermissionThrowing(AbstractActionInput actionInput, String processName, Map processValues) throws QPermissionDeniedException { - QProcessMetaData process = QContext.getQInstance().getProcess(processName); + QProcessMetaData process = QContext.getQInstance().getProcess(processName); + + if(process == null) + { + LOG.info("Throwing a permission denied exception in response to a non-existent process name", logPair("processName", processName)); + throw (new QPermissionDeniedException("Permission denied.")); + } + QPermissionRules effectivePermissionRules = getEffectivePermissionRules(process, QContext.getQInstance()); if(effectivePermissionRules.getCustomPermissionChecker() != null) @@ -226,6 +240,13 @@ public class PermissionsHelper public static void checkAppPermissionThrowing(AbstractActionInput actionInput, String appName) throws QPermissionDeniedException { QAppMetaData app = QContext.getQInstance().getApp(appName); + + if(app == null) + { + LOG.info("Throwing a permission denied exception in response to a non-existent app name", logPair("appName", appName)); + throw (new QPermissionDeniedException("Permission denied.")); + } + commonCheckPermissionThrowing(getEffectivePermissionRules(app, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, app.getName()); } @@ -255,6 +276,13 @@ public class PermissionsHelper public static void checkReportPermissionThrowing(AbstractActionInput actionInput, String reportName) throws QPermissionDeniedException { QReportMetaData report = QContext.getQInstance().getReport(reportName); + + if(report == null) + { + LOG.info("Throwing a permission denied exception in response to a non-existent process name", logPair("reportName", reportName)); + throw (new QPermissionDeniedException("Permission denied.")); + } + commonCheckPermissionThrowing(getEffectivePermissionRules(report, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, report.getName()); } @@ -284,6 +312,13 @@ public class PermissionsHelper public static void checkWidgetPermissionThrowing(AbstractActionInput actionInput, String widgetName) throws QPermissionDeniedException { QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(widgetName); + + if(widget == null) + { + LOG.info("Throwing a permission denied exception in response to a non-existent widget name", logPair("widgetName", widgetName)); + throw (new QPermissionDeniedException("Permission denied.")); + } + commonCheckPermissionThrowing(getEffectivePermissionRules(widget, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, widget.getName()); } From a9a988f2210bab8c80a536205b3b9f27cff6251a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 10:23:47 -0500 Subject: [PATCH 08/19] Add missing overloads for debug,warn,error(LogPair ...) --- .../qqq/backend/core/logging/QLogger.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java index 42bfe93d..b14b0e1e 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java @@ -280,6 +280,16 @@ public class QLogger + /******************************************************************************* + ** + *******************************************************************************/ + public void debug(LogPair... logPairs) + { + logger.warn(() -> makeJsonString(null, null, logPairs)); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -420,6 +430,16 @@ public class QLogger + /******************************************************************************* + ** + *******************************************************************************/ + public void warn(LogPair... logPairs) + { + logger.warn(() -> makeJsonString(null, null, logPairs)); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -480,6 +500,16 @@ public class QLogger + /******************************************************************************* + ** + *******************************************************************************/ + public void error(LogPair... logPairs) + { + logger.warn(() -> makeJsonString(null, null, logPairs)); + } + + + /******************************************************************************* ** *******************************************************************************/ From 576ca8a6dfe8ea4e80651d91fdb28acce7cbd1e4 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 10:24:39 -0500 Subject: [PATCH 09/19] Add withCriteria overloads that match most common constructor signatures for QFilterCriteria --- .../actions/tables/query/QQueryFilter.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java index dba1a93e..6de05ddc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -237,6 +238,28 @@ public class QQueryFilter implements Serializable, Cloneable + /******************************************************************************* + ** fluent method to add a new criteria + *******************************************************************************/ + public QQueryFilter withCriteria(String fieldName, QCriteriaOperator operator, Collection values) + { + addCriteria(new QFilterCriteria(fieldName, operator, values)); + return (this); + } + + + + /******************************************************************************* + ** fluent method to add a new criteria + *******************************************************************************/ + public QQueryFilter withCriteria(String fieldName, QCriteriaOperator operator, Serializable... values) + { + addCriteria(new QFilterCriteria(fieldName, operator, values)); + return (this); + } + + + /******************************************************************************* ** *******************************************************************************/ From c2a13b1adaf6add60f4f280a172f79e61b119b33 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 10:26:11 -0500 Subject: [PATCH 10/19] Expose orderInstructionsJoinOrder on order table; flip orderInstructionsJoinOrder (to expose bug covered in testFlippedJoinForOnClause --- .../com/kingsrook/qqq/backend/module/rdbms/TestUtils.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java index 2570f6ea..354f91f8 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java @@ -273,6 +273,7 @@ public class TestUtils .withJoinNameChain(List.of("orderInstructionsJoinOrder"))) .withField(new QFieldMetaData("orderId", QFieldType.INTEGER).withBackendName("order_id")) .withField(new QFieldMetaData("instructions", QFieldType.STRING)) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER).withJoinPath(List.of("orderInstructionsJoinOrder"))) ); qInstance.addTable(defineBaseTable(TABLE_NAME_ITEM, "item") @@ -395,8 +396,8 @@ public class TestUtils qInstance.addJoin(new QJoinMetaData() .withName("orderInstructionsJoinOrder") - .withLeftTable(TABLE_NAME_ORDER_INSTRUCTIONS) - .withRightTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_ORDER_INSTRUCTIONS) + .withLeftTable(TABLE_NAME_ORDER) .withType(JoinType.MANY_TO_ONE) .withJoinOn(new JoinOn("orderId", "id")) ); From a3433d60f7e99c5a90c9370e47e90efe65b9cd71 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 10:27:16 -0500 Subject: [PATCH 11/19] CE-1406 remove tests that weren't ready for commit --- .../actions/RDBMSQueryActionJoinsTest.java | 120 ------------------ 1 file changed, 120 deletions(-) diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java index 22480177..44dfbad0 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java @@ -26,14 +26,11 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; -import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; @@ -50,7 +47,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; -import com.kingsrook.qqq.backend.core.utils.collections.SetBuilder; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -953,76 +949,6 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testMultipleReversedDirectionJoinsBetweenSameTables() throws QException - { - ///////////////////////////////////////////////////////////////////////////////////////////////// - // primer sql file loads 2 instructions for order 1, 3 for order 2, then 1 for all the others. // - // let's make things a bit more interesting by deleting the 2 for order 1 // - ///////////////////////////////////////////////////////////////////////////////////////////////// - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - DeleteInput deleteInput = new DeleteInput(); - deleteInput.setTableName(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS); - deleteInput.setQueryFilter(new QQueryFilter(new QFilterCriteria("orderId", QCriteriaOperator.EQUALS, 1))); - new DeleteAction().execute(deleteInput); - - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); - - // Integer noOfOrders = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER)).getCount(); - // Integer noOfOrderInstructions = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS)).getCount(); - - { - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // make sure we can join on order.current_order_instruction_id = order_instruction.id -- and that we get back 1 row per order // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS).withSelect(true).withJoinMetaData(QContext.getQInstance().getJoin("orderJoinCurrentOrderInstructions"))); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - assertEquals(2, queryOutput.getRecords().size()); - } - - { - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // make sure we can join on order.current_order_instruction_id = order_instruction.id -- and that we get back 1 row per order // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS).withSelect(true).withType(QueryJoin.Type.LEFT).withJoinMetaData(QContext.getQInstance().getJoin("orderJoinCurrentOrderInstructions"))); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - assertEquals(5, queryOutput.getRecords().size()); - } - - /* - { - //////////////////////////////////////////////////////////////////////////////////////////////// - // assert that the query succeeds (based on exposed join) if the joinMetaData isn't specified // - //////////////////////////////////////////////////////////////////////////////////////////////// - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS)); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - // assertEquals(noOfOrders, queryOutput.getRecords().size()); - } - - { - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // make sure we can join on order.id = order_instruction.order_id (e.g., not the exposed one used above) -- and that we get back 1 row per order instruction // - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS).withJoinMetaData(QContext.getQInstance().getJoin("orderInstructionsJoinOrder"))); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - // assertEquals(noOfOrderInstructions, queryOutput.getRecords().size()); - } - */ - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -1126,50 +1052,4 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest assertEquals(5, new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON)).getRecords().size()); } - - - /******************************************************************************* - ** scenario: - ** order LEFT JOIN shipment. both have storeId as security field. - ** not all orders have a shipment (but due to left-join, expect to get all - ** orders back, even if missing a shipment). - ** - ** If the security clause for shipment is in the WHERE clause, it will "negate" - ** the effect of the LEFT JOIN (unless it includes, e.g., "OR id IS NULL"). - ** - ** At one point, we tried AND'ing such a security clause to the JOIN ... ON, - ** but that is frequently causing issues... - *******************************************************************************/ - @Test - void testLeftJoinSecurityClause() throws QException - { - ////////////////////////////////////////////////////////////////////////////////// - // scenario: // - // order LEFT JOIN shipment. both have storeId as security field. // - // not all orders have a shipment (but due to left-join, expect to get all // - // orders back, even if missing a shipment). // - // // - // If the security clause for shipment is in the WHERE clause, it will "negate" // - // the effect of the LEFT JOIN (unless it includes, e.g., "OR id IS NULL"). // - // // - // At one point, we tried AND'ing such a security clause to the JOIN ... ON, // - // but that is frequently causing issues... // - // // - // order table has 3 rows for store 1: // - // - id=1 has 1 shipment, but assigned to the wrong store! // - // - id=2 has no shipments. // - // - id=3 has 2 shipments. // - ////////////////////////////////////////////////////////////////////////////////// - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); - - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHIPMENT).withSelect(true).withType(QueryJoin.Type.LEFT)); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - List records = queryOutput.getRecords(); - assertEquals(4, records.size(), "expected no of records"); - assertEquals(SetBuilder.of(1), records.stream().map(r -> r.getValue("storeId")).collect(Collectors.toSet())); - assertEquals(SetBuilder.of(null, 1), records.stream().map(r -> r.getValue(TestUtils.TABLE_NAME_SHIPMENT + ".storeId")).collect(Collectors.toSet())); - } - } From 27c693f0c4e925281ca9f8f4e84f57308e859477 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 10:56:51 -0500 Subject: [PATCH 12/19] CE-1406 Fix orderInstructionsJoinOrder --- .../java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java index 354f91f8..7a84cc63 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java @@ -399,7 +399,7 @@ public class TestUtils .withRightTable(TABLE_NAME_ORDER_INSTRUCTIONS) .withLeftTable(TABLE_NAME_ORDER) .withType(JoinType.MANY_TO_ONE) - .withJoinOn(new JoinOn("orderId", "id")) + .withJoinOn(new JoinOn("id", "orderId")) ); qInstance.addPossibleValueSource(new QPossibleValueSource() From 27a6c0d53c2c218030aff1e24b02219cfc13ebd5 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 14:34:00 -0500 Subject: [PATCH 13/19] CE-1406 in ensureRecordSecurityLockIsRepresented, getTable using table name, not a (potential) alias; avoid NPE on exposedJoins; whitespace; add cloneable in JoinOn --- .../actions/tables/query/JoinsContext.java | 9 +++++---- .../core/model/metadata/joins/JoinOn.java | 20 ++++++++++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java index 26767cce..7f2adde6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java @@ -378,7 +378,7 @@ public class JoinsContext { securityFieldTableAlias = matchedQueryJoin.getJoinTableOrItsAlias(); } - tmpTable = instance.getTable(securityFieldTableAlias); + tmpTable = instance.getTable(aliasToTableNameMap.getOrDefault(securityFieldTableAlias, securityFieldTableAlias)); //////////////////////////////////////////////////////////////////////////////////////// // set the baseTableOrAlias for the next iteration to be this join's joinTableOrAlias // @@ -466,8 +466,8 @@ public class JoinsContext ////////////////////////////////////////////////////////////////////////////////////////////////////////////// // check if the key type has an all-access key, and if so, if it's set to true for the current user/session // ////////////////////////////////////////////////////////////////////////////////////////////////////////////// - QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType()); - boolean haveAllAccessKey = false; + QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType()); + boolean haveAllAccessKey = false; if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName())) { ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -1118,7 +1118,7 @@ public class JoinsContext if(useExposedJoins) { QTableMetaData mainTable = QContext.getQInstance().getTable(mainTableName); - for(ExposedJoin exposedJoin : mainTable.getExposedJoins()) + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(mainTable.getExposedJoins())) { if(exposedJoin.getJoinTable().equals(joinTableName)) { @@ -1159,6 +1159,7 @@ public class JoinsContext } + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/JoinOn.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/JoinOn.java index 0dc8ed98..78666893 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/JoinOn.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/JoinOn.java @@ -26,7 +26,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.joins; ** Specification for (at least part of) how two tables join together - e.g., ** leftField = rightField. Used as part of a list in a QJoinMetaData. *******************************************************************************/ -public class JoinOn +public class JoinOn implements Cloneable { private String leftField; private String rightField; @@ -131,4 +131,22 @@ public class JoinOn return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public JoinOn clone() + { + try + { + JoinOn clone = (JoinOn) super.clone(); + return clone; + } + catch(CloneNotSupportedException e) + { + throw new AssertionError(); + } + } } From 1a6cc5bf3c1b3202ca16d55f0a219340c07c03fa Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 14:35:49 -0500 Subject: [PATCH 14/19] CE-1406 Add Cloneable --- .../model/actions/tables/query/QueryJoin.java | 36 ++++++++++++++++- .../model/metadata/joins/QJoinMetaData.java | 40 ++++++++++++++++++- 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryJoin.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryJoin.java index b55fbea2..d7a433cc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryJoin.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryJoin.java @@ -56,7 +56,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; ** JoinsContext is constructed before executing a query, and not meant to be set ** by users. *******************************************************************************/ -public class QueryJoin +public class QueryJoin implements Cloneable { private String baseTableOrAlias; private String joinTable; @@ -69,6 +69,40 @@ public class QueryJoin + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryJoin clone() + { + try + { + QueryJoin clone = (QueryJoin) super.clone(); + + if(joinMetaData != null) + { + clone.joinMetaData = joinMetaData.clone(); + } + + if(securityCriteria != null) + { + clone.securityCriteria = new ArrayList<>(); + for(QFilterCriteria securityCriterion : securityCriteria) + { + clone.securityCriteria.add(securityCriterion.clone()); + } + } + + return clone; + } + catch(CloneNotSupportedException e) + { + throw new AssertionError(); + } + } + + + /******************************************************************************* ** define the types of joins - INNER, LEFT, RIGHT, or FULL. *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/QJoinMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/QJoinMetaData.java index 1f35d165..51c8dd30 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/QJoinMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/QJoinMetaData.java @@ -33,7 +33,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* ** Definition of how 2 tables join together within a QQQ Instance. *******************************************************************************/ -public class QJoinMetaData implements TopLevelMetaDataInterface +public class QJoinMetaData implements TopLevelMetaDataInterface, Cloneable { private String name; private JoinType type; @@ -62,6 +62,44 @@ public class QJoinMetaData implements TopLevelMetaDataInterface + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QJoinMetaData clone() + { + try + { + QJoinMetaData clone = (QJoinMetaData) super.clone(); + + if(joinOns != null) + { + clone.joinOns = new ArrayList<>(); + for(JoinOn joinOn : joinOns) + { + clone.joinOns.add(joinOn.clone()); + } + } + + if(orderBys != null) + { + clone.orderBys = new ArrayList<>(); + for(QFilterOrderBy orderBy : orderBys) + { + clone.orderBys.add(orderBy.clone()); + } + } + + return clone; + } + catch(CloneNotSupportedException e) + { + throw new AssertionError(); + } + } + + + /******************************************************************************* ** Getter for name ** From 95998b687b988c91454992d483d97d9c93ac2a08 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 14:35:59 -0500 Subject: [PATCH 15/19] CE-1406 Add renderedReportId to output --- .../savedreports/RenderSavedReportExecuteStep.java | 1 + 1 file changed, 1 insertion(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java index 506d2857..f91a9000 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java @@ -129,6 +129,7 @@ public class RenderSavedReportExecuteStep implements BackendStep .withRenderedReportStatusId(RenderedReportStatus.RUNNING.getId()) .withReportFormat(ReportFormatPossibleValueEnum.valueOf(reportFormat.name()).getPossibleValueId()) )).getRecords().get(0); + runBackendStepOutput.addValue("renderedReportId", renderedReportRecord.getValue("id")); //////////////////////////////////////////////////////////////////////////////////////////// // convert the report record to report meta-data, which the GenerateReportAction works on // From 099fd273097774b0e8d975a8d7d885b9aac7c55a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 14:39:44 -0500 Subject: [PATCH 16/19] CE-1406 Initial checkin --- .../ReportsFullInstanceVerifier.java | 258 ++++++++++++++++++ .../ColumnStatsFullInstanceVerifierTest.java | 48 ++++ .../ReportsFullInstanceVerifierTest.java | 61 +++++ 3 files changed, 367 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifier.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifierTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifierTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifier.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifier.java new file mode 100644 index 00000000..60c2f509 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifier.java @@ -0,0 +1,258 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.savedreports.RenderedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Utility for verifying that the RenderReports process works for all fields, + ** on all tables, and all exposed joins. + ** + ** Meant for use within a unit test, or maybe as part of an instance's boot-up/ + ** validation. + *******************************************************************************/ +public class ReportsFullInstanceVerifier +{ + private static final QLogger LOG = QLogger.getLogger(ReportsFullInstanceVerifier.class); + + private boolean removeRenderedReports = true; + private boolean filterForAtMostOneRowPerReport = true; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void verify(Collection tables, String storageTableName) throws QException + { + Map, Exception> caughtExceptions = new LinkedHashMap<>(); + for(QTableMetaData table : tables) + { + if(table.isCapabilityEnabled(QContext.getQInstance().getBackendForTable(table.getName()), Capability.TABLE_QUERY)) + { + LOG.info("Verifying Reports on table", logPair("tableName", table.getName())); + + ////////////////////////////////////////////// + // run the table by itself (no join fields) // + ////////////////////////////////////////////// + runReport(table.getName(), Collections.emptyList(), "main-table-only", caughtExceptions, storageTableName); + + /////////////////////////////////////////////////// + // run once w/ the fields from each exposed join // + /////////////////////////////////////////////////// + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + runReport(table.getName(), List.of(exposedJoin), "join-" + exposedJoin.getLabel(), caughtExceptions, storageTableName); + } + + ///////////////////////////////////////////////// + // run w/ all exposed joins (if there are any) // + ///////////////////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(table.getExposedJoins())) + { + runReport(table.getName(), table.getExposedJoins(), "all-joins", caughtExceptions, storageTableName); + } + } + } + + ////////////////////////////////// + // log out an exceptions caught // + ////////////////////////////////// + if(!caughtExceptions.isEmpty()) + { + for(Map.Entry, Exception> entry : caughtExceptions.entrySet()) + { + LOG.info("Caught an exception verifying reports", entry.getValue(), logPair("tableName", entry.getKey().getA()), logPair("fieldName", entry.getKey().getB())); + } + throw (new QException("Reports Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size()))); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void runReport(String tableName, List exposedJoinList, String description, Map, Exception> caughtExceptions, String storageTableName) + { + try + { + //////////////////////////////////////////////////////////////////////////////////////////////// + // build the list of reports to include in the column - starting with all fields in the table // + //////////////////////////////////////////////////////////////////////////////////////////////// + ReportColumns reportColumns = new ReportColumns(); + for(QFieldMetaData field : QContext.getQInstance().getTable(tableName).getFields().values()) + { + reportColumns.withColumn(field.getName()); + } + + /////////////////////////////////////////////////// + // add all fields from all exposed joins as well // + /////////////////////////////////////////////////// + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(exposedJoinList)) + { + QTableMetaData joinTable = QContext.getQInstance().getTable(exposedJoin.getJoinTable()); + for(QFieldMetaData field : joinTable.getFields().values()) + { + reportColumns.withColumn(joinTable.getName() + "." + field.getName()); + } + } + + QQueryFilter queryFilter = new QQueryFilter(); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if caller is okay with a filter that should limit the report to a small number of rows (could be more than 1 for to-many joins), then do so // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(filterForAtMostOneRowPerReport) + { + queryFilter.withCriteria(QContext.getQInstance().getTable(tableName).getPrimaryKeyField(), QCriteriaOperator.EQUALS, 1); + } + + ////////////////////////////////// + // insert a saved report record // + ////////////////////////////////// + SavedReport savedReport = new SavedReport(); + savedReport.setTableName(tableName); + savedReport.setLabel("Test " + tableName + " " + description); + savedReport.setColumnsJson(JsonUtils.toJson(reportColumns)); + savedReport.setQueryFilterJson(JsonUtils.toJson(queryFilter)); + List reportRecordList = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(savedReport)).getRecords(); + + /////////////////////// + // render the report // + /////////////////////// + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + + input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT, ReportFormat.CSV.name()); + input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME, storageTableName); + input.setRecords(reportRecordList); + + new RenderSavedReportExecuteStep().run(input, output); + + ////////////////////////////////////////// + // clean up the report, if so requested // + ////////////////////////////////////////// + if(removeRenderedReports) + { + new DeleteAction().execute(new DeleteInput(RenderedReport.TABLE_NAME).withPrimaryKey(output.getValue("renderedReportId"))); + } + } + catch(QException e) + { + caughtExceptions.put(Pair.of(tableName, description), e); + } + } + + /******************************************************************************* + ** Getter for removeRenderedReports + *******************************************************************************/ + public boolean getRemoveRenderedReports() + { + return (this.removeRenderedReports); + } + + + + /******************************************************************************* + ** Setter for removeRenderedReports + *******************************************************************************/ + public void setRemoveRenderedReports(boolean removeRenderedReports) + { + this.removeRenderedReports = removeRenderedReports; + } + + + + /******************************************************************************* + ** Fluent setter for removeRenderedReports + *******************************************************************************/ + public ReportsFullInstanceVerifier withRemoveRenderedReports(boolean removeRenderedReports) + { + this.removeRenderedReports = removeRenderedReports; + return (this); + } + + + + /******************************************************************************* + ** Getter for filterForAtMostOneRowPerReport + *******************************************************************************/ + public boolean getFilterForAtMostOneRowPerReport() + { + return (this.filterForAtMostOneRowPerReport); + } + + + + /******************************************************************************* + ** Setter for filterForAtMostOneRowPerReport + *******************************************************************************/ + public void setFilterForAtMostOneRowPerReport(boolean filterForAtMostOneRowPerReport) + { + this.filterForAtMostOneRowPerReport = filterForAtMostOneRowPerReport; + } + + + + /******************************************************************************* + ** Fluent setter for filterForAtMostOneRowPerReport + *******************************************************************************/ + public ReportsFullInstanceVerifier withFilterForAtMostOneRowPerReport(boolean filterForAtMostOneRowPerReport) + { + this.filterForAtMostOneRowPerReport = filterForAtMostOneRowPerReport; + return (this); + } + + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifierTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifierTest.java new file mode 100644 index 00000000..b2476f9b --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifierTest.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.columnstats; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for ColumnStatsFullInstanceVerifier + *******************************************************************************/ +class ColumnStatsFullInstanceVerifierTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + new ColumnStatsFullInstanceVerifier().verify(List.of(QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY))); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifierTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifierTest.java new file mode 100644 index 00000000..a3730f28 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifierTest.java @@ -0,0 +1,61 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for ReportsFullInstanceVerifier + *******************************************************************************/ +class ReportsFullInstanceVerifierTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws Exception + { + new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + new ReportsFullInstanceVerifier().verify(List.of(QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)), SavedReportsMetaDataProvider.REPORT_STORAGE_TABLE_NAME); + } + +} \ No newline at end of file From 31fa3c3921dc0f697a476633f4802ebdc7815696 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Jul 2024 15:19:33 -0500 Subject: [PATCH 17/19] CE-1406 Update to clone queryJoins... since our friend the JoinContext likes to mutate them, and break things! also cleaned up all warnings. --- .../reporting/GenerateReportAction.java | 66 ++++++++++++------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java index a984bed3..a216e9e5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java @@ -66,6 +66,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; @@ -303,7 +304,7 @@ public class GenerateReportAction extends AbstractQActionFunction cloneDataSourceQueryJoins(QReportDataSource dataSource) + { + if(dataSource == null || dataSource.getQueryJoins() == null) + { + return (null); + } + + List rs = new ArrayList<>(); + for(QueryJoin queryJoin : dataSource.getQueryJoins()) + { + rs.add(queryJoin.clone()); + } + return (rs); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -417,12 +438,12 @@ public class GenerateReportAction extends AbstractQActionFunction setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext) throws QException + private Set setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource) throws QException { Set fieldsToTranslatePossibleValues = new HashSet<>(); @@ -574,9 +595,9 @@ public class GenerateReportAction extends AbstractQActionFunction records, QReportView tableView, List summaryViews, List variantViews) throws QException + private Integer consumeRecords(QReportDataSource dataSource, List records, QReportView tableView, List summaryViews, List variantViews) throws QException { - QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable()); + QTableMetaData table = QContext.getQInstance().getTable(dataSource.getSourceTable()); //////////////////////////////////////////////////////////////////////////// // if this record goes on a table view, add it to the report streamer now // @@ -687,7 +708,7 @@ public class GenerateReportAction extends AbstractQActionFunction>> viewAggregates, SummaryKey key) throws QException + private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map>> viewAggregates, SummaryKey key) { Map> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>()); addRecordToAggregatesMap(table, record, keyAggregates); @@ -698,7 +719,7 @@ public class GenerateReportAction extends AbstractQActionFunction> aggregatesMap) throws QException + private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map> aggregatesMap) { ////////////////////////////////////////////////////////////////////////////////////// // todo - an optimization could be, to only compute aggregates that we'll need... // @@ -706,7 +727,7 @@ public class GenerateReportAction extends AbstractQActionFunction reportViews = views.stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList(); for(QReportView view : reportViews) { - QReportDataSource dataSource = getDataSource(view.getDataSourceName()); - QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable()); - SummaryOutput summaryOutput = computeSummaryRowsForView(reportInput, view, table); + QReportDataSource dataSource = getDataSource(view.getDataSourceName()); + if(dataSource == null) + { + throw new QReportingException("Data source for summary view was not found (viewName=" + view.getName() + ", dataSourceName=" + view.getDataSourceName() + ")."); + } + + QTableMetaData table = QContext.getQInstance().getTable(dataSource.getSourceTable()); + SummaryOutput summaryOutput = computeSummaryRowsForView(reportInput, view, table); ExportInput exportInput = new ExportInput(); exportInput.setReportDestination(reportInput.getReportDestination()); @@ -867,9 +893,8 @@ public class GenerateReportAction extends AbstractQActionFunction - { - return summaryRowComparator(view, o1, o2); - }); + summaryRows.sort((o1, o2) -> summaryRowComparator(view, o1, o2)); } //////////////// @@ -979,8 +1001,6 @@ public class GenerateReportAction extends AbstractQActionFunction Date: Tue, 9 Jul 2024 11:03:21 -0500 Subject: [PATCH 18/19] CE-1406 Initial checkin --- .../ExportsFullInstanceVerifier.java | 202 ++++++++++++++++++ .../ExportsFullInstanceVerifierTest.java | 46 ++++ 2 files changed, 248 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifier.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifierTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifier.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifier.java new file mode 100644 index 00000000..bd4c3dfa --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifier.java @@ -0,0 +1,202 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Utility for verifying that the ExportAction works for all tables, and all + ** exposed joins. + ** + ** Meant for use within a unit test, or maybe as part of an instance's boot-up/ + ** validation. + *******************************************************************************/ +public class ExportsFullInstanceVerifier +{ + private static final QLogger LOG = QLogger.getLogger(ExportsFullInstanceVerifier.class); + + private boolean filterForAtMostOneRowPerExport = true; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void verify(Collection tables) throws QException + { + Map, Exception> caughtExceptions = new LinkedHashMap<>(); + for(QTableMetaData table : tables) + { + if(table.isCapabilityEnabled(QContext.getQInstance().getBackendForTable(table.getName()), Capability.TABLE_QUERY)) + { + LOG.info("Verifying Exports on table", logPair("tableName", table.getName())); + + ////////////////////////////////////////////// + // run the table by itself (no join fields) // + ////////////////////////////////////////////// + runExport(table.getName(), Collections.emptyList(), "main-table-only", caughtExceptions); + + /////////////////////////////////////////////////// + // run once w/ the fields from each exposed join // + /////////////////////////////////////////////////// + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + runExport(table.getName(), List.of(exposedJoin), "join-" + exposedJoin.getLabel(), caughtExceptions); + } + + ///////////////////////////////////////////////// + // run w/ all exposed joins (if there are any) // + ///////////////////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(table.getExposedJoins())) + { + runExport(table.getName(), table.getExposedJoins(), "all-joins", caughtExceptions); + } + } + } + + ////////////////////////////////// + // log out an exceptions caught // + ////////////////////////////////// + if(!caughtExceptions.isEmpty()) + { + for(Map.Entry, Exception> entry : caughtExceptions.entrySet()) + { + LOG.info("Caught an exception verifying reports", entry.getValue(), logPair("tableName", entry.getKey().getA()), logPair("fieldName", entry.getKey().getB())); + } + throw (new QException("Reports Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size()))); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void runExport(String tableName, List exposedJoinList, String description, Map, Exception> caughtExceptions) + { + try + { + //////////////////////////////////////////////////////////////////////////////////// + // build the list of fieldNames to export - starting with all fields in the table // + //////////////////////////////////////////////////////////////////////////////////// + List fieldNames = new ArrayList<>(); + for(QFieldMetaData field : QContext.getQInstance().getTable(tableName).getFields().values()) + { + fieldNames.add(field.getName()); + } + + /////////////////////////////////////////////////// + // add all fields from all exposed joins as well // + /////////////////////////////////////////////////// + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(exposedJoinList)) + { + QTableMetaData joinTable = QContext.getQInstance().getTable(exposedJoin.getJoinTable()); + for(QFieldMetaData field : joinTable.getFields().values()) + { + fieldNames.add(joinTable.getName() + "." + field.getName()); + } + } + + LOG.info("Verifying export", logPair("description", description), logPair("fieldCount", fieldNames.size())); + + QQueryFilter queryFilter = new QQueryFilter(); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if caller is okay with a filter that should limit the report to a small number of rows (could be more than 1 for to-many joins), then do so // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(filterForAtMostOneRowPerExport) + { + queryFilter.withCriteria(QContext.getQInstance().getTable(tableName).getPrimaryKeyField(), QCriteriaOperator.EQUALS, 1); + } + + ExportInput exportInput = new ExportInput(); + exportInput.setTableName(tableName); + exportInput.setFieldNames(fieldNames); + exportInput.setReportDestination(new ReportDestination() + .withReportOutputStream(new ByteArrayOutputStream()) + .withReportFormat(ReportFormat.CSV)); + exportInput.setQueryFilter(queryFilter); + new ExportAction().execute(exportInput); + } + catch(QException e) + { + caughtExceptions.put(Pair.of(tableName, description), e); + } + } + + + + /******************************************************************************* + ** Getter for filterForAtMostOneRowPerExport + *******************************************************************************/ + public boolean getFilterForAtMostOneRowPerExport() + { + return (this.filterForAtMostOneRowPerExport); + } + + + + /******************************************************************************* + ** Setter for filterForAtMostOneRowPerExport + *******************************************************************************/ + public void setFilterForAtMostOneRowPerExport(boolean filterForAtMostOneRowPerExport) + { + this.filterForAtMostOneRowPerExport = filterForAtMostOneRowPerExport; + } + + + + /******************************************************************************* + ** Fluent setter for filterForAtMostOneRowPerExport + *******************************************************************************/ + public ExportsFullInstanceVerifier withFilterForAtMostOneRowPerExport(boolean filterForAtMostOneRowPerExport) + { + this.filterForAtMostOneRowPerExport = filterForAtMostOneRowPerExport; + return (this); + } + + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifierTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifierTest.java new file mode 100644 index 00000000..bf3e28b7 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifierTest.java @@ -0,0 +1,46 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for ExportsFullInstanceVerifier + *******************************************************************************/ +class ExportsFullInstanceVerifierTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + new ExportsFullInstanceVerifier().verify(QContext.getQInstance().getTables().values()); + } + +} \ No newline at end of file From eb36630bcd2a57930d03332d755c82dd123c9ac0 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 9 Jul 2024 11:34:56 -0500 Subject: [PATCH 19/19] CE-1406 Initial checkin --- .../SavedReportsTableFullVerifier.java | 151 ++++++++++++++++++ .../SavedReportsTableFullVerifierTest.java | 84 ++++++++++ 2 files changed, 235 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifier.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifierTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifier.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifier.java new file mode 100644 index 00000000..d6e7e687 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifier.java @@ -0,0 +1,151 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedreports.RenderedReport; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Utility for verifying that the RenderReports process works for all report + ** records stored in the saved reports table. + ** + ** Meant for use within a unit test, or maybe as part of an instance's boot-up/ + ** validation. + *******************************************************************************/ +public class SavedReportsTableFullVerifier +{ + private static final QLogger LOG = QLogger.getLogger(SavedReportsTableFullVerifier.class); + + private boolean removeRenderedReports = true; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void verify(List savedReportRecordList, String storageTableName) throws QException + { + Map caughtExceptions = new LinkedHashMap<>(); + for(QRecord report : savedReportRecordList) + { + runReport(report, caughtExceptions, storageTableName); + } + + ////////////////////////////////// + // log out an exceptions caught // + ////////////////////////////////// + if(!caughtExceptions.isEmpty()) + { + for(Map.Entry entry : caughtExceptions.entrySet()) + { + LOG.info("Caught an exception verifying saved reports", entry.getValue(), logPair("savdReportId", entry.getKey())); + } + throw (new QException("Saved Reports Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size()))); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void runReport(QRecord savedReport, Map caughtExceptions, String storageTableName) + { + try + { + /////////////////////// + // render the report // + /////////////////////// + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + + input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT, ReportFormat.XLSX.name()); + input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME, storageTableName); + input.setRecords(List.of(savedReport)); + + new RenderSavedReportExecuteStep().run(input, output); + Exception exception = output.getException(); + if(exception != null) + { + throw (exception); + } + + ////////////////////////////////////////// + // clean up the report, if so requested // + ////////////////////////////////////////// + if(removeRenderedReports) + { + new DeleteAction().execute(new DeleteInput(RenderedReport.TABLE_NAME).withPrimaryKey(output.getValue("renderedReportId"))); + } + } + catch(Exception e) + { + caughtExceptions.put(savedReport.getValueInteger("id"), e); + } + } + + + + /******************************************************************************* + ** Getter for removeRenderedReports + *******************************************************************************/ + public boolean getRemoveRenderedReports() + { + return (this.removeRenderedReports); + } + + + + /******************************************************************************* + ** Setter for removeRenderedReports + *******************************************************************************/ + public void setRemoveRenderedReports(boolean removeRenderedReports) + { + this.removeRenderedReports = removeRenderedReports; + } + + + + /******************************************************************************* + ** Fluent setter for removeRenderedReports + *******************************************************************************/ + public SavedReportsTableFullVerifier withRemoveRenderedReports(boolean removeRenderedReports) + { + this.removeRenderedReports = removeRenderedReports; + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifierTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifierTest.java new file mode 100644 index 00000000..431bf0a6 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifierTest.java @@ -0,0 +1,84 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for SavedReportsTableFullVerifier + *******************************************************************************/ +class SavedReportsTableFullVerifierTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws Exception + { + new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + GenerateReportActionTest.insertPersonRecords(QContext.getQInstance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + ReportColumns reportColumns = new ReportColumns(); + reportColumns.withColumn("id"); + + ////////////////////////////////// + // insert a saved report record // + ////////////////////////////////// + SavedReport savedReport = new SavedReport(); + savedReport.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + savedReport.setLabel("Test"); + savedReport.setColumnsJson(JsonUtils.toJson(reportColumns)); + savedReport.setQueryFilterJson(JsonUtils.toJson(new QQueryFilter())); + List reportRecordList = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(savedReport)).getRecords(); + + SavedReportsTableFullVerifier savedReportsTableFullVerifier = new SavedReportsTableFullVerifier(); + savedReportsTableFullVerifier.verify(reportRecordList, SavedReportsMetaDataProvider.REPORT_STORAGE_TABLE_NAME); + } + +} \ No newline at end of file