From 88e47ef9ca44a5c92512ed07faadd11a7f03fd3d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Apr 2023 10:04:56 -0500 Subject: [PATCH 01/15] WIP on getting joins to frontend --- .../core/actions/metadata/JoinGraph.java | 233 ++++++++++++++++++ .../core/actions/metadata/MetaDataAction.java | 152 ++++++++++++ 2 files changed, 385 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java new file mode 100644 index 00000000..decb9ee4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java @@ -0,0 +1,233 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.metadata; + + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class JoinGraph +{ + private record Node(String tableName) + { + } + + + + private record Edge(String joinName, String leftTable, String rightTable) + { + } + + + + private static class CanonicalJoin + { + private String tableA; + private String tableB; + private String joinFieldA; + private String joinFieldB; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public CanonicalJoin(QJoinMetaData joinMetaData) + { + boolean needFlip = false; + int tableCompare = joinMetaData.getLeftTable().compareTo(joinMetaData.getRightTable()); + if(tableCompare < 0) + { + needFlip = true; + } + else if(tableCompare == 0) + { + int fieldCompare = joinMetaData.getJoinOns().get(0).getLeftField().compareTo(joinMetaData.getJoinOns().get(0).getRightField()); + if(fieldCompare < 0) + { + needFlip = true; + } + } + + if(needFlip) + { + joinMetaData = joinMetaData.flip(); + } + + tableA = joinMetaData.getLeftTable(); + tableB = joinMetaData.getRightTable(); + joinFieldA = joinMetaData.getJoinOns().get(0).getLeftField(); + joinFieldB = joinMetaData.getJoinOns().get(0).getRightField(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean equals(Object o) + { + if(this == o) + { + return true; + } + if(o == null || getClass() != o.getClass()) + { + return false; + } + CanonicalJoin that = (CanonicalJoin) o; + return Objects.equals(tableA, that.tableA) && Objects.equals(tableB, that.tableB) && Objects.equals(joinFieldA, that.joinFieldA) && Objects.equals(joinFieldB, that.joinFieldB); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int hashCode() + { + return Objects.hash(tableA, tableB, joinFieldA, joinFieldB); + } + } + + + + private Set nodes = new HashSet<>(); + private Set edges = new HashSet<>(); + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public JoinGraph(QInstance qInstance) + { + Set usedJoins = new HashSet<>(); + for(QJoinMetaData join : qInstance.getJoins().values()) + { + CanonicalJoin canonicalJoin = new CanonicalJoin(join); + if(usedJoins.contains(canonicalJoin)) + { + continue; + } + + usedJoins.add(canonicalJoin); + nodes.add(new Node(join.getLeftTable())); + nodes.add(new Node(join.getRightTable())); + edges.add(new Edge(join.getName(), join.getLeftTable(), join.getRightTable())); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Set getJoins(String tableName) + { + Set rs = new HashSet<>(); + Set tables = new HashSet<>(); + tables.add(tableName); + + boolean keepGoing = true; + while(keepGoing) + { + keepGoing = false; + for(Edge edge : edges) + { + if(tables.contains(edge.leftTable) || tables.contains(edge.rightTable)) + { + if(!rs.contains(edge.joinName)) + { + tables.add(edge.leftTable); + tables.add(edge.rightTable); + rs.add(edge.joinName); + keepGoing = true; + } + } + } + } + + return (rs); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public record Something(String joinTable, List joinPath) + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Set getJoinsBetter(String tableName) + { + Set rs = new HashSet<>(); + Set usedEdges = new HashSet<>(); + Set tables = new HashSet<>(); + tables.add(tableName); + doGetJoinsBetter(rs, tables, new ArrayList<>(), usedEdges); + + return (rs); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void doGetJoinsBetter(Set rs, Set tables, List joinPath, Set usedEdges) + { + for(Edge edge : edges) + { + if(usedEdges.contains(edge.joinName)) + { + continue; + } + + if(tables.contains(edge.leftTable) || tables.contains(edge.rightTable)) + { + usedEdges.add(edge.joinName); + // todo - clone list here, then recurisiv call + rs.add(new Something(tables.contains(edge.leftTable) ? edge.rightTable : edge.leftTable, joinPath)); + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java index 04dbc905..6d2d70eb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java @@ -23,16 +23,21 @@ package com.kingsrook.qqq.backend.core.actions.metadata; import java.util.ArrayList; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNode; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendAppMetaData; @@ -40,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendProcessMe import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; @@ -88,6 +94,9 @@ public class MetaDataAction } metaDataOutput.setTables(tables); + // addJoinsToTables(tables); + // addJoinedTablesToTables(tables); + //////////////////////////////////////// // map processes to frontend metadata // //////////////////////////////////////// @@ -212,6 +221,149 @@ public class MetaDataAction return metaDataOutput; } + ////////////////////////////////////// start v1 ////////////////////////////////////// + + + + private record JoinedTable(String joinedTableName, List joinPath) + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void addJoinedTablesToTables(Map tables) + { + for(QFrontendTableMetaData table : tables.values()) + { + List joinedTables = new ArrayList<>(); + addJoinedTablesToTable(tables, table, joinedTables, new ArrayList<>()); + + if(joinedTables.size() > 0) + { + System.out.println("For [" + table.getName() + "] we have:\n " + joinedTables.stream().map(String::valueOf).collect(Collectors.joining("\n ")) + "\n"); + } + else + { + System.out.println("No joins for [" + table.getName() + "]\n"); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void addJoinedTablesToTable(Map tables, QFrontendTableMetaData table, List joinedTables, List joinPath) + { + QInstance qInstance = QContext.getQInstance(); + for(QJoinMetaData join : qInstance.getJoins().values()) + { + if(join.getLeftTable().equals(table.getName())) + { + String joinName = join.getName(); + JoinedTable joinedTable = new JoinedTable(join.getRightTable(), joinPath); + System.out.println("Adding to [" + table.getName() + "]: " + joinedTable); + joinedTables.add(joinedTable); + + ArrayList subJoinPath = new ArrayList<>(joinPath); + subJoinPath.add(joinName); + addJoinedTablesToTable(tables, tables.get(join.getRightTable()), joinedTables, subJoinPath); + } + if(join.getRightTable().equals(table.getName())) + { + String joinName = join.getName() + ".flipped"; + JoinedTable joinedTable = new JoinedTable(join.getLeftTable(), joinPath); + System.out.println("Adding to [" + table.getName() + "]: " + joinedTable); + joinedTables.add(joinedTable); + + ArrayList subJoinPath = new ArrayList<>(joinPath); + subJoinPath.add(joinName); + addJoinedTablesToTable(tables, tables.get(join.getLeftTable()), joinedTables, subJoinPath); + } + } + } + + ////////////////////////////////////// end v1 ////////////////////////////////////// + + ////////////////////////////////////// start v0 ////////////////////////////////////// + + + + private record Something(String joinName, List joinPath) + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void addJoinsToTables(Map tables) + { + for(QFrontendTableMetaData table : tables.values()) + { + List something = new ArrayList<>(); + addJoinsToTable(tables, table, something, new ArrayList<>(), new HashSet<>()); + if(something.size() > 0) + { + System.out.println("For [" + table.getName() + "] we have:\n " + something.stream().map(String::valueOf).collect(Collectors.joining("\n ")) + "\n"); + } + else + { + System.out.println("No joins for [" + table.getName() + "]\n"); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void addJoinsToTable(Map tables, QFrontendTableMetaData table, List something, List joinPath, Set usedJoins) + { + QInstance qInstance = QContext.getQInstance(); + for(QJoinMetaData join : qInstance.getJoins().values()) + { + if(join.getLeftTable().equals(table.getName())) + { + String joinName = join.getName(); + if(!usedJoins.contains(joinName)) + { + usedJoins.add(joinName); + something.add(new Something(joinName, joinPath)); + + ArrayList subJoinPath = new ArrayList<>(joinPath); + subJoinPath.add(joinName); + + QFrontendTableMetaData rightTable = tables.get(join.getRightTable()); + addJoinsToTable(tables, rightTable, something, subJoinPath, usedJoins); + } + } + else if(join.getRightTable().equals(table.getName())) + { + String joinName = join.getName() + ".flipped"; + if(!usedJoins.contains(joinName)) + { + usedJoins.add(joinName); + something.add(new Something(joinName, joinPath)); + + ArrayList subJoinPath = new ArrayList<>(joinPath); + subJoinPath.add(joinName); + + QFrontendTableMetaData leftTable = tables.get(join.getLeftTable()); + addJoinsToTable(tables, leftTable, something, subJoinPath, usedJoins); + } + } + } + } + + ////////////////////////////////////// end v0 ////////////////////////////////////// + /******************************************************************************* From 3bf18e8b51ce75f29276c1537d0d84a13e1fe791 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Apr 2023 10:16:03 -0500 Subject: [PATCH 02/15] Initial checkin --- .../core/actions/metadata/JoinGraphTest.java | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraphTest.java diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraphTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraphTest.java new file mode 100644 index 00000000..344a8c3b --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraphTest.java @@ -0,0 +1,58 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.metadata; + + +import java.util.Set; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class JoinGraphTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + JoinGraph joinGraph = new JoinGraph(QContext.getQInstance()); + for(QTableMetaData table : QContext.getQInstance().getTables().values()) + { + Set joins = joinGraph.getJoins(table.getName()); + if(joins.isEmpty()) + { + System.out.println(table.getName() + " has no joins"); + } + else + { + System.out.println(table.getName() + " joins: " + joins); + } + } + } + +} From 6ce5845ec8d4fa0df9906e02d41c141ab9f8a5e8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Sun, 16 Apr 2023 19:50:58 -0500 Subject: [PATCH 03/15] Checkpoint - good version of getJoinConnections now i think --- .../core/actions/metadata/JoinGraph.java | 263 ++++++++++++++++++ 1 file changed, 263 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java index decb9ee4..26f6bccc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java @@ -23,10 +23,12 @@ package com.kingsrook.qqq.backend.core.actions.metadata; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.TreeSet; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; @@ -184,6 +186,267 @@ public class JoinGraph + public record JoinConnection(String joinTable, String viaJoinName) implements Comparable + { + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int compareTo(JoinConnection that) + { + Comparator comparator = Comparator.comparing((JoinConnection jc) -> jc.joinTable()) + .thenComparing((JoinConnection jc) -> jc.viaJoinName()); + return (comparator.compare(this, that)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public record JoinConnectionList(List list) implements Comparable + { + + /******************************************************************************* + ** + *******************************************************************************/ + public JoinConnectionList copy() + { + return new JoinConnectionList(new ArrayList<>(list)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public int compareTo(JoinConnectionList that) + { + if(this.equals(that)) + { + return (0); + } + + for(int i = 0; i < Math.min(this.list.size(), that.list.size()); i++) + { + int comp = this.list.get(i).compareTo(that.list.get(i)); + if(comp != 0) + { + return (comp); + } + } + + return (this.list.size() - that.list.size()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Set getJoinConnections(String tableName) + { + Set rs = new TreeSet<>(); + doGetJoinConnections(rs, tableName, new ArrayList<>(), new JoinConnectionList(new ArrayList<>())); + return (rs); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void doGetJoinConnections(Set joinConnections, String tableName, List path, JoinConnectionList connectionList) + { + for(Edge edge : edges) + { + if(edge.leftTable.equals(tableName) || edge.rightTable.equals(tableName)) + { + if(path.contains(edge.joinName)) + { + continue; + } + + List newPath = new ArrayList<>(path); + newPath.add(edge.joinName); + if(!joinConnectionsContain(joinConnections, newPath)) + { + String otherTableName = null; + if(!edge.leftTable.equals(tableName)) + { + otherTableName = edge.leftTable; + } + else if(!edge.rightTable.equals(tableName)) + { + otherTableName = edge.rightTable; + } + + if(otherTableName != null) + { + + JoinConnectionList newConnectionList = connectionList.copy(); + JoinConnection joinConnection = new JoinConnection(otherTableName, edge.joinName); + newConnectionList.list.add(joinConnection); + joinConnections.add(newConnectionList); + doGetJoinConnections(joinConnections, otherTableName, new ArrayList<>(newPath), newConnectionList); + } + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean joinConnectionsContain(Set joinPaths, List newPath) + { + for(JoinConnectionList joinConnections : joinPaths) + { + List joinConnectionJoins = joinConnections.list.stream().map(jc -> jc.viaJoinName).toList(); + if(joinConnectionJoins.equals(newPath)) + { + return (true); + } + } + return (false); + } + + + + public record JoinPath(String joinTable, List joinNames) + { + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Set getJoinPaths(String tableName) + { + Set rs = new HashSet<>(); + doGetJoinPaths(rs, tableName, new ArrayList<>()); + return (rs); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void doGetJoinPaths(Set joinPaths, String tableName, List path) + { + for(Edge edge : edges) + { + if(edge.leftTable.equals(tableName) || edge.rightTable.equals(tableName)) + { + if(path.contains(edge.joinName)) + { + continue; + } + + List newPath = new ArrayList<>(path); + newPath.add(edge.joinName); + if(!joinPathsContain(joinPaths, newPath)) + { + String otherTableName = null; + if(!edge.leftTable.equals(tableName)) + { + otherTableName = edge.leftTable; + } + else if(!edge.rightTable.equals(tableName)) + { + otherTableName = edge.rightTable; + } + + if(otherTableName != null) + { + joinPaths.add(new JoinPath(otherTableName, newPath)); + doGetJoinPaths(joinPaths, otherTableName, new ArrayList<>(newPath)); + } + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean joinPathsContain(Set joinPaths, List newPath) + { + for(JoinPath joinPath : joinPaths) + { + if(joinPath.joinNames().equals(newPath)) + { + return (true); + } + } + return (false); + } + + /******************************************************************************* + ** + *******************************************************************************/ + /* + public Set> getJoinPaths(String tableName) + { + Set> rs = new HashSet<>(); + doGetJoinPaths(rs, tableName, new ArrayList<>()); + return (rs); + } + */ + + /******************************************************************************* + ** + *******************************************************************************/ + /* + private void doGetJoinPaths(Set> joinPaths, String tableName, List path) + { + for(Edge edge : edges) + { + if(edge.leftTable.equals(tableName) || edge.rightTable.equals(tableName)) + { + if(path.contains(edge.joinName)) + { + continue; + } + + List newPath = new ArrayList<>(path); + newPath.add(edge.joinName); + if(!joinPaths.contains(newPath)) + { + joinPaths.add(newPath); + + String otherTableName = null; + if(!edge.leftTable.equals(tableName)) + { + otherTableName = edge.leftTable; + } + else if(!edge.rightTable.equals(tableName)) + { + otherTableName = edge.rightTable; + } + + if(otherTableName != null) + { + doGetJoinPaths(joinPaths, otherTableName, new ArrayList<>(newPath)); + } + } + } + } + } + */ + + + /******************************************************************************* ** *******************************************************************************/ From d086284de78923525f70b91e97127b1841422afd Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 20 Apr 2023 09:35:12 -0500 Subject: [PATCH 04/15] Checkpoint --- .../core/instances/QInstanceEnricher.java | 74 ++++++++++ .../model/metadata/tables/ExposedJoin.java | 130 ++++++++++++++++++ .../model/metadata/tables/QTableMetaData.java | 49 +++++++ 3 files changed, 253 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoin.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 8746b4a1..70185db0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -32,7 +32,9 @@ import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph; import com.kingsrook.qqq.backend.core.actions.permissions.BulkTableActionProcessPermissionChecker; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -59,6 +61,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QMiddlewareTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -144,6 +147,77 @@ public class QInstanceEnricher { qInstance.getWidgets().values().forEach(this::enrichWidget); } + + if(CollectionUtils.nullSafeHasContents(qInstance.getJoins())) + { + //todo! enrichJoins(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void enrichJoins() + { + try + { + JoinGraph joinGraph = new JoinGraph(qInstance); + + for(QTableMetaData table : qInstance.getTables().values()) + { + Set joinConnections = joinGraph.getJoinConnections(table.getName()); + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // proceed with caution - remember, validator will fail the instance if things are missing/invalid // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + if(exposedJoin.getJoinTable() != null) + { + QTableMetaData joinTable = qInstance.getTable(exposedJoin.getJoinTable()); + if(joinTable != null) + { + ////////////////////////////////////////////////////////////////////////////////// + // default the exposed join's label to the join table's label, if it wasn't set // + ////////////////////////////////////////////////////////////////////////////////// + if(!StringUtils.hasContent(exposedJoin.getLabel())) + { + exposedJoin.setLabel(joinTable.getLabel()); + } + + /////////////////////////////////////////////////////////////////////////////// + // default the exposed join's join-path from the joinGraph, if it wasn't set // + /////////////////////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeIsEmpty(exposedJoin.getJoinPath())) + { + List eligibleJoinConnections = new ArrayList<>(); + for(JoinGraph.JoinConnectionList joinConnection : joinConnections) + { + if(joinTable.getName().equals(joinConnection.list().get(0).joinTable())) + { + eligibleJoinConnections.add(joinConnection); + } + } + + if(eligibleJoinConnections.isEmpty()) + { + throw (new QException("Could not infer a joinPath for table [" + table.getName() + "], exposedJoin to [" + exposedJoin.getJoinTable() + "] - no join connections exist in this instance.")); + } + else if(eligibleJoinConnections.size() > 1) + { + throw (new QException("Could not infer a joinPath for table [" + table.getName() + "], exposedJoin to [" + exposedJoin.getJoinTable() + "] - multiple possible join connections exist in this instance: ")); // todo - list the paths so user can choose one! + } + } + } + } + } + } + } + catch(Exception e) + { + throw (new RuntimeException("Error enriching instance joins", e)); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoin.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoin.java new file mode 100644 index 00000000..6588cb21 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoin.java @@ -0,0 +1,130 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.util.List; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ExposedJoin +{ + private String label; + private String joinTable; + private List joinPath; + + + + /******************************************************************************* + ** Getter for label + *******************************************************************************/ + public String getLabel() + { + return (this.label); + } + + + + /******************************************************************************* + ** Setter for label + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + *******************************************************************************/ + public ExposedJoin withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for joinTable + *******************************************************************************/ + public String getJoinTable() + { + return (this.joinTable); + } + + + + /******************************************************************************* + ** Setter for joinTable + *******************************************************************************/ + public void setJoinTable(String joinTable) + { + this.joinTable = joinTable; + } + + + + /******************************************************************************* + ** Fluent setter for joinTable + *******************************************************************************/ + public ExposedJoin withJoinTable(String joinTable) + { + this.joinTable = joinTable; + return (this); + } + + + + /******************************************************************************* + ** Getter for joinPath + *******************************************************************************/ + public List getJoinPath() + { + return (this.joinPath); + } + + + + /******************************************************************************* + ** Setter for joinPath + *******************************************************************************/ + public void setJoinPath(List joinPath) + { + this.joinPath = joinPath; + } + + + + /******************************************************************************* + ** Fluent setter for joinPath + *******************************************************************************/ + public ExposedJoin withJoinPath(List joinPath) + { + this.joinPath = joinPath; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index 77249eef..2fa71424 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -101,6 +101,8 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData private Map middlewareMetaData; + private List exposedJoins; + /******************************************************************************* @@ -1296,4 +1298,51 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData { qInstance.addTable(this); } + + + + /******************************************************************************* + ** Getter for exposedJoins + *******************************************************************************/ + public List getExposedJoins() + { + return (this.exposedJoins); + } + + + + /******************************************************************************* + ** Setter for exposedJoins + *******************************************************************************/ + public void setExposedJoins(List exposedJoins) + { + this.exposedJoins = exposedJoins; + } + + + + /******************************************************************************* + ** Fluent setter for exposedJoins + *******************************************************************************/ + public QTableMetaData withExposedJoins(List exposedJoins) + { + this.exposedJoins = exposedJoins; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for exposedJoins + *******************************************************************************/ + public QTableMetaData withExposedJoin(ExposedJoin exposedJoin) + { + if(this.exposedJoins == null) + { + this.exposedJoins = new ArrayList<>(); + } + this.exposedJoins.add(exposedJoin); + return (this); + } + } From 249598958482d04be2aa5c9f74d440021e608ca3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Apr 2023 12:11:46 -0500 Subject: [PATCH 05/15] add exposed joins to frontend metadata; checkpoing on validation & enrichment of eposed joins --- .../core/actions/metadata/JoinGraph.java | 292 ++++-------------- .../core/actions/metadata/MetaDataAction.java | 2 +- .../actions/metadata/TableMetaDataAction.java | 2 +- .../core/instances/QInstanceEnricher.java | 33 +- .../core/instances/QInstanceValidator.java | 70 ++++- .../core/model/metadata/QInstance.java | 29 ++ .../frontend/QFrontendExposedJoin.java | 178 +++++++++++ .../frontend/QFrontendTableMetaData.java | 41 ++- .../model/metadata/tables/ExposedJoin.java | 78 +++++ .../core/actions/metadata/JoinGraphTest.java | 58 ---- .../core/instances/QInstanceEnricherTest.java | 132 ++++++++ .../instances/QInstanceValidatorTest.java | 81 +++++ .../metadata/tables/ExposedJoinTest.java | 137 ++++++++ 13 files changed, 833 insertions(+), 300 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendExposedJoin.java delete mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraphTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoinTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java index 26f6bccc..fc597ed7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java @@ -31,26 +31,37 @@ import java.util.Set; import java.util.TreeSet; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* - ** + ** Object to represent the graph of joins in a QQQ Instance. e.g., all of the + ** connections among tables through joins. *******************************************************************************/ public class JoinGraph { - private record Node(String tableName) - { - } + + private Set edges = new HashSet<>(); + /******************************************************************************* + ** Graph edge (no graph nodes needed in here) + *******************************************************************************/ private record Edge(String joinName, String leftTable, String rightTable) { } - private static class CanonicalJoin + /******************************************************************************* + ** In this class, we are treating joins as non-directional graph edges - so - + ** use this class to "normalize" what may otherwise be duplicated joins in the + ** qInstance (e.g., A -> B and B -> A -- in the instance, those are valid, but + ** in our graph here, we want to consider those the same). + *******************************************************************************/ + private static class NormalizedJoin { private String tableA; private String tableB; @@ -62,7 +73,7 @@ public class JoinGraph /******************************************************************************* ** *******************************************************************************/ - public CanonicalJoin(QJoinMetaData joinMetaData) + public NormalizedJoin(QJoinMetaData joinMetaData) { boolean needFlip = false; int tableCompare = joinMetaData.getLeftTable().compareTo(joinMetaData.getRightTable()); @@ -106,7 +117,7 @@ public class JoinGraph { return false; } - CanonicalJoin that = (CanonicalJoin) o; + NormalizedJoin that = (NormalizedJoin) o; return Objects.equals(tableA, that.tableA) && Objects.equals(tableB, that.tableB) && Objects.equals(joinFieldA, that.joinFieldA) && Objects.equals(joinFieldB, that.joinFieldB); } @@ -124,29 +135,22 @@ public class JoinGraph - private Set nodes = new HashSet<>(); - private Set edges = new HashSet<>(); - - - /******************************************************************************* ** Constructor ** *******************************************************************************/ public JoinGraph(QInstance qInstance) { - Set usedJoins = new HashSet<>(); - for(QJoinMetaData join : qInstance.getJoins().values()) + Set usedJoins = new HashSet<>(); + for(QJoinMetaData join : CollectionUtils.nonNullMap(qInstance.getJoins()).values()) { - CanonicalJoin canonicalJoin = new CanonicalJoin(join); - if(usedJoins.contains(canonicalJoin)) + NormalizedJoin normalizedJoin = new NormalizedJoin(join); + if(usedJoins.contains(normalizedJoin)) { continue; } - usedJoins.add(canonicalJoin); - nodes.add(new Node(join.getLeftTable())); - nodes.add(new Node(join.getRightTable())); + usedJoins.add(normalizedJoin); edges.add(new Edge(join.getName(), join.getLeftTable(), join.getRightTable())); } } @@ -156,36 +160,6 @@ public class JoinGraph /******************************************************************************* ** *******************************************************************************/ - public Set getJoins(String tableName) - { - Set rs = new HashSet<>(); - Set tables = new HashSet<>(); - tables.add(tableName); - - boolean keepGoing = true; - while(keepGoing) - { - keepGoing = false; - for(Edge edge : edges) - { - if(tables.contains(edge.leftTable) || tables.contains(edge.rightTable)) - { - if(!rs.contains(edge.joinName)) - { - tables.add(edge.leftTable); - tables.add(edge.rightTable); - rs.add(edge.joinName); - keepGoing = true; - } - } - } - } - - return (rs); - } - - - public record JoinConnection(String joinTable, String viaJoinName) implements Comparable { @@ -240,6 +214,49 @@ public class JoinGraph return (this.list.size() - that.list.size()); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean matchesJoinPath(List joinPath) + { + if(list.size() != joinPath.size()) + { + return (false); + } + + for(int i = 0; i < list.size(); i++) + { + if(!list.get(i).viaJoinName().equals(joinPath.get(i))) + { + return (false); + } + } + + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public String getJoinNamesAsString() + { + return (StringUtils.join(", ", list().stream().map(jc -> jc.viaJoinName()).toList())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public List getJoinNamesAsList() + { + return (list().stream().map(jc -> jc.viaJoinName()).toList()); + } } @@ -316,181 +333,4 @@ public class JoinGraph return (false); } - - - public record JoinPath(String joinTable, List joinNames) - { - - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public Set getJoinPaths(String tableName) - { - Set rs = new HashSet<>(); - doGetJoinPaths(rs, tableName, new ArrayList<>()); - return (rs); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void doGetJoinPaths(Set joinPaths, String tableName, List path) - { - for(Edge edge : edges) - { - if(edge.leftTable.equals(tableName) || edge.rightTable.equals(tableName)) - { - if(path.contains(edge.joinName)) - { - continue; - } - - List newPath = new ArrayList<>(path); - newPath.add(edge.joinName); - if(!joinPathsContain(joinPaths, newPath)) - { - String otherTableName = null; - if(!edge.leftTable.equals(tableName)) - { - otherTableName = edge.leftTable; - } - else if(!edge.rightTable.equals(tableName)) - { - otherTableName = edge.rightTable; - } - - if(otherTableName != null) - { - joinPaths.add(new JoinPath(otherTableName, newPath)); - doGetJoinPaths(joinPaths, otherTableName, new ArrayList<>(newPath)); - } - } - } - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private boolean joinPathsContain(Set joinPaths, List newPath) - { - for(JoinPath joinPath : joinPaths) - { - if(joinPath.joinNames().equals(newPath)) - { - return (true); - } - } - return (false); - } - - /******************************************************************************* - ** - *******************************************************************************/ - /* - public Set> getJoinPaths(String tableName) - { - Set> rs = new HashSet<>(); - doGetJoinPaths(rs, tableName, new ArrayList<>()); - return (rs); - } - */ - - /******************************************************************************* - ** - *******************************************************************************/ - /* - private void doGetJoinPaths(Set> joinPaths, String tableName, List path) - { - for(Edge edge : edges) - { - if(edge.leftTable.equals(tableName) || edge.rightTable.equals(tableName)) - { - if(path.contains(edge.joinName)) - { - continue; - } - - List newPath = new ArrayList<>(path); - newPath.add(edge.joinName); - if(!joinPaths.contains(newPath)) - { - joinPaths.add(newPath); - - String otherTableName = null; - if(!edge.leftTable.equals(tableName)) - { - otherTableName = edge.leftTable; - } - else if(!edge.rightTable.equals(tableName)) - { - otherTableName = edge.rightTable; - } - - if(otherTableName != null) - { - doGetJoinPaths(joinPaths, otherTableName, new ArrayList<>(newPath)); - } - } - } - } - } - */ - - - - /******************************************************************************* - ** - *******************************************************************************/ - public record Something(String joinTable, List joinPath) - { - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public Set getJoinsBetter(String tableName) - { - Set rs = new HashSet<>(); - Set usedEdges = new HashSet<>(); - Set tables = new HashSet<>(); - tables.add(tableName); - doGetJoinsBetter(rs, tables, new ArrayList<>(), usedEdges); - - return (rs); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void doGetJoinsBetter(Set rs, Set tables, List joinPath, Set usedEdges) - { - for(Edge edge : edges) - { - if(usedEdges.contains(edge.joinName)) - { - continue; - } - - if(tables.contains(edge.leftTable) || tables.contains(edge.rightTable)) - { - usedEdges.add(edge.joinName); - // todo - clone list here, then recurisiv call - rs.add(new Something(tables.contains(edge.leftTable) ? edge.rightTable : edge.leftTable, joinPath)); - } - } - } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java index 6d2d70eb..f0bde491 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java @@ -89,7 +89,7 @@ public class MetaDataAction } QBackendMetaData backendForTable = metaDataInput.getInstance().getBackendForTable(tableName); - tables.put(tableName, new QFrontendTableMetaData(metaDataInput, backendForTable, table, false)); + tables.put(tableName, new QFrontendTableMetaData(metaDataInput, backendForTable, table, false, false)); treeNodes.put(tableName, new AppTreeNode(table)); } metaDataOutput.setTables(tables); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/TableMetaDataAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/TableMetaDataAction.java index 223eb475..46c4f243 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/TableMetaDataAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/TableMetaDataAction.java @@ -54,7 +54,7 @@ public class TableMetaDataAction throw (new QNotFoundException("Table [" + tableMetaDataInput.getTableName() + "] was not found.")); } QBackendMetaData backendForTable = tableMetaDataInput.getInstance().getBackendForTable(table.getName()); - tableMetaDataOutput.setTable(new QFrontendTableMetaData(tableMetaDataInput, backendForTable, table, true)); + tableMetaDataOutput.setTable(new QFrontendTableMetaData(tableMetaDataInput, backendForTable, table, true, true)); // todo post-customization - can do whatever w/ the result if you want diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 70185db0..6820042f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -90,6 +90,8 @@ public class QInstanceEnricher private final QInstance qInstance; + private JoinGraph joinGraph; + ////////////////////////////////////////////////////////// // todo - come up w/ a way for app devs to set configs! // ////////////////////////////////////////////////////////// @@ -148,10 +150,7 @@ public class QInstanceEnricher qInstance.getWidgets().values().forEach(this::enrichWidget); } - if(CollectionUtils.nullSafeHasContents(qInstance.getJoins())) - { - //todo! enrichJoins(); - } + enrichJoins(); } @@ -163,9 +162,9 @@ public class QInstanceEnricher { try { - JoinGraph joinGraph = new JoinGraph(qInstance); + joinGraph = new JoinGraph(qInstance); - for(QTableMetaData table : qInstance.getTables().values()) + for(QTableMetaData table : CollectionUtils.nonNullMap(qInstance.getTables()).values()) { Set joinConnections = joinGraph.getJoinConnections(table.getName()); for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) @@ -194,7 +193,7 @@ public class QInstanceEnricher List eligibleJoinConnections = new ArrayList<>(); for(JoinGraph.JoinConnectionList joinConnection : joinConnections) { - if(joinTable.getName().equals(joinConnection.list().get(0).joinTable())) + if(joinTable.getName().equals(joinConnection.list().get(joinConnection.list().size() - 1).joinTable())) { eligibleJoinConnections.add(joinConnection); } @@ -202,11 +201,18 @@ public class QInstanceEnricher if(eligibleJoinConnections.isEmpty()) { - throw (new QException("Could not infer a joinPath for table [" + table.getName() + "], exposedJoin to [" + exposedJoin.getJoinTable() + "] - no join connections exist in this instance.")); + throw (new QException("Could not infer a joinPath for table [" + table.getName() + "], exposedJoin to [" + exposedJoin.getJoinTable() + "]: No join connections between these tables exist in this instance.")); } else if(eligibleJoinConnections.size() > 1) { - throw (new QException("Could not infer a joinPath for table [" + table.getName() + "], exposedJoin to [" + exposedJoin.getJoinTable() + "] - multiple possible join connections exist in this instance: ")); // todo - list the paths so user can choose one! + throw (new QException("Could not infer a joinPath for table [" + table.getName() + "], exposedJoin to [" + exposedJoin.getJoinTable() + "]: " + + eligibleJoinConnections.size() + " join connections exist between these tables. You need to specify one:\n" + + StringUtils.join("\n", eligibleJoinConnections.stream().map(jcl -> jcl.getJoinNamesAsString()).toList()) + "." + )); + } + else + { + exposedJoin.setJoinPath(eligibleJoinConnections.get(0).getJoinNamesAsList()); } } } @@ -1089,4 +1095,13 @@ public class QInstanceEnricher } } + + + /******************************************************************************* + ** + *******************************************************************************/ + public JoinGraph getJoinGraph() + { + return (this.joinGraph); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 7dfa6963..938b5944 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -36,6 +36,7 @@ import java.util.function.Supplier; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface; import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; @@ -69,6 +70,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; @@ -115,17 +117,26 @@ public class QInstanceValidator return; } + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // the enricher will build a join graph (if there are any joins). we'd like to only do that // + // once, during the enrichment/validation work, so, capture it, and store it back in the instance. // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + JoinGraph joinGraph = null; try { ///////////////////////////////////////////////////////////////////////////////////////////////// // before validation, enrich the object (e.g., to fill in values that the user doesn't have to // ///////////////////////////////////////////////////////////////////////////////////////////////// // TODO - possible point of customization (use a different enricher, or none, or pass it options). - new QInstanceEnricher(qInstance).enrich(); + QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(qInstance); + qInstanceEnricher.enrich(); + joinGraph = qInstanceEnricher.getJoinGraph(); } catch(Exception e) { + System.out.println(); LOG.error("Error enriching instance prior to validation", e); + System.out.println(); throw (new QInstanceValidationException("Error enriching qInstance prior to validation.", e)); } @@ -136,7 +147,7 @@ public class QInstanceValidator { validateBackends(qInstance); validateAutomationProviders(qInstance); - validateTables(qInstance); + validateTables(qInstance, joinGraph); validateProcesses(qInstance); validateReports(qInstance); validateApps(qInstance); @@ -158,7 +169,9 @@ public class QInstanceValidator throw (new QInstanceValidationException(errors)); } - qInstance.setHasBeenValidated(new QInstanceValidationKey()); + QInstanceValidationKey validationKey = new QInstanceValidationKey(); + qInstance.setHasBeenValidated(validationKey); + qInstance.setJoinGraph(validationKey, joinGraph); } @@ -366,7 +379,7 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateTables(QInstance qInstance) + private void validateTables(QInstance qInstance, JoinGraph joinGraph) { if(assertCondition(CollectionUtils.nullSafeHasContents(qInstance.getTables()), "At least 1 table must be defined.")) { @@ -459,12 +472,61 @@ public class QInstanceValidator validateTableCacheOf(qInstance, table); validateTableRecordSecurityLocks(qInstance, table); validateTableAssociations(qInstance, table); + validateExposedJoins(qInstance, joinGraph, table); }); } } + /******************************************************************************* + ** + *******************************************************************************/ + private void validateExposedJoins(QInstance qInstance, JoinGraph joinGraph, QTableMetaData table) + { + Set joinConnectionsForTable = null; + Set usedLabels = new HashSet<>(); + Set> usedJoinPaths = new HashSet<>(); + + String tablePrefix = "Table " + table.getName() + " "; + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + String joinPrefix = tablePrefix + "exposedJoin [missingJoinTableName] "; + if(assertCondition(StringUtils.hasContent(exposedJoin.getJoinTable()), tablePrefix + "has an exposedJoin that is missing a joinTable name.")) + { + joinPrefix = tablePrefix + "exposedJoin " + exposedJoin.getJoinTable() + " "; + if(assertCondition(qInstance.getTable(exposedJoin.getJoinTable()) != null, joinPrefix + "is referencing an unrecognized table")) + { + if(assertCondition(CollectionUtils.nullSafeHasContents(exposedJoin.getJoinPath()), joinPrefix + "is missing a joinPath.")) + { + joinConnectionsForTable = Objects.requireNonNullElseGet(joinConnectionsForTable, () -> joinGraph.getJoinConnections(table.getName())); + + boolean foundJoinConnection = false; + for(JoinGraph.JoinConnectionList joinConnectionList : joinConnectionsForTable) + { + if(joinConnectionList.matchesJoinPath(exposedJoin.getJoinPath())) + { + foundJoinConnection = true; + } + } + assertCondition(foundJoinConnection, joinPrefix + "specified a joinPath [" + exposedJoin.getJoinPath() + "] which does not match a valid join connection in the instance."); + + assertCondition(!usedJoinPaths.contains(exposedJoin.getJoinPath()), tablePrefix + "has more than one join with the joinPath: " + exposedJoin.getJoinPath()); + usedJoinPaths.add(exposedJoin.getJoinPath()); + } + } + } + + if(assertCondition(StringUtils.hasContent(exposedJoin.getLabel()), joinPrefix + "is missing a label.")) + { + assertCondition(!usedLabels.contains(exposedJoin.getLabel()), tablePrefix + "has more than one join labeled: " + exposedJoin.getLabel()); + usedLabels.add(exposedJoin.getLabel()); + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index be3b7074..0b9f45ee 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph; import com.kingsrook.qqq.backend.core.actions.metadata.MetaDataAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.instances.QInstanceValidationKey; @@ -106,6 +107,8 @@ public class QInstance private Map memoizedTablePaths = new HashMap<>(); private Map memoizedProcessPaths = new HashMap<>(); + private JoinGraph joinGraph; + /******************************************************************************* @@ -1136,4 +1139,30 @@ public class QInstance this.middlewareMetaData.put(middlewareMetaData.getType(), middlewareMetaData); return (this); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public JoinGraph getJoinGraph() + { + return (this.joinGraph); + } + + + + /******************************************************************************* + ** Only the validation (and enrichment) code should set the instance's joinGraph + ** so, we take a package-only-constructable validation key as a param along with + ** the joinGraph - and we throw IllegalArgumentException if a non-null key is given. + *******************************************************************************/ + public void setJoinGraph(QInstanceValidationKey key, JoinGraph joinGraph) throws IllegalArgumentException + { + if(key == null) + { + throw (new IllegalArgumentException("A ValidationKey must be provided")); + } + this.joinGraph = joinGraph; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendExposedJoin.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendExposedJoin.java new file mode 100644 index 00000000..f89816c8 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendExposedJoin.java @@ -0,0 +1,178 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.frontend; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; + + +/******************************************************************************* + ** Version of an ExposedJoin for a frontend to see + *******************************************************************************/ +public class QFrontendExposedJoin +{ + private String label; + private Boolean isMany; + private QFrontendTableMetaData joinTable; + private List joinPath; + + + + /******************************************************************************* + ** Getter for label + *******************************************************************************/ + public String getLabel() + { + return (this.label); + } + + + + /******************************************************************************* + ** Setter for label + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + *******************************************************************************/ + public QFrontendExposedJoin withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for joinTable + *******************************************************************************/ + public QFrontendTableMetaData getJoinTable() + { + return (this.joinTable); + } + + + + /******************************************************************************* + ** Setter for joinTable + *******************************************************************************/ + public void setJoinTable(QFrontendTableMetaData joinTable) + { + this.joinTable = joinTable; + } + + + + /******************************************************************************* + ** Fluent setter for joinTable + *******************************************************************************/ + public QFrontendExposedJoin withJoinTable(QFrontendTableMetaData joinTable) + { + this.joinTable = joinTable; + return (this); + } + + + + /******************************************************************************* + ** Getter for joinPath + *******************************************************************************/ + public List getJoinPath() + { + return (this.joinPath); + } + + + + /******************************************************************************* + ** Setter for joinPath + *******************************************************************************/ + public void setJoinPath(List joinPath) + { + this.joinPath = joinPath; + } + + + + /******************************************************************************* + ** Fluent setter for joinPath + *******************************************************************************/ + public QFrontendExposedJoin withJoinPath(List joinPath) + { + this.joinPath = joinPath; + return (this); + } + + + + /******************************************************************************* + ** Add one join to the join path in here + *******************************************************************************/ + public void addJoin(QJoinMetaData join) + { + if(this.joinPath == null) + { + this.joinPath = new ArrayList<>(); + } + this.joinPath.add(join); + } + + + + /******************************************************************************* + ** Getter for isMany + *******************************************************************************/ + public Boolean getIsMany() + { + return (this.isMany); + } + + + + /******************************************************************************* + ** Setter for isMany + *******************************************************************************/ + public void setIsMany(Boolean isMany) + { + this.isMany = isMany; + } + + + + /******************************************************************************* + ** Fluent setter for isMany + *******************************************************************************/ + public QFrontendExposedJoin withIsMany(Boolean isMany) + { + this.isMany = isMany; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java index c53cc933..199d42a6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.frontend; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -32,12 +33,16 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; /******************************************************************************* @@ -58,6 +63,8 @@ public class QFrontendTableMetaData private Map fields; private List sections; + private List exposedJoins; + private Set capabilities; private boolean readPermission; @@ -74,7 +81,7 @@ public class QFrontendTableMetaData /******************************************************************************* ** *******************************************************************************/ - public QFrontendTableMetaData(AbstractActionInput actionInput, QBackendMetaData backendForTable, QTableMetaData tableMetaData, boolean includeFields) + public QFrontendTableMetaData(AbstractActionInput actionInput, QBackendMetaData backendForTable, QTableMetaData tableMetaData, boolean includeFields, boolean includeJoins) { this.name = tableMetaData.getName(); this.label = tableMetaData.getLabel(); @@ -92,6 +99,27 @@ public class QFrontendTableMetaData this.sections = tableMetaData.getSections(); } + if(includeJoins) + { + QInstance qInstance = QContext.getQInstance(); + + this.exposedJoins = new ArrayList<>(); + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(tableMetaData.getExposedJoins())) + { + QFrontendExposedJoin frontendExposedJoin = new QFrontendExposedJoin(); + this.exposedJoins.add(frontendExposedJoin); + + QTableMetaData joinTable = qInstance.getTable(exposedJoin.getJoinTable()); + frontendExposedJoin.setLabel(exposedJoin.getLabel()); + frontendExposedJoin.setIsMany(exposedJoin.getIsMany()); + frontendExposedJoin.setJoinTable(new QFrontendTableMetaData(actionInput, backendForTable, joinTable, includeFields, false)); + for(String joinName : exposedJoin.getJoinPath()) + { + frontendExposedJoin.addJoin(qInstance.getJoin(joinName)); + } + } + } + if(tableMetaData.getIcon() != null) { this.iconName = tableMetaData.getIcon().getName(); @@ -259,4 +287,15 @@ public class QFrontendTableMetaData { return deletePermission; } + + + + /******************************************************************************* + ** Getter for exposedJoins + ** + *******************************************************************************/ + public List getExposedJoins() + { + return exposedJoins; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoin.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoin.java index 6588cb21..f66b1b23 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoin.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoin.java @@ -23,6 +23,12 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables; import java.util.List; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; /******************************************************************************* @@ -30,10 +36,17 @@ import java.util.List; *******************************************************************************/ public class ExposedJoin { + private static final QLogger LOG = QLogger.getLogger(ExposedJoin.class); + private String label; private String joinTable; private List joinPath; + ////////////////////////////////////////////////////////////////// + // no setter for this - derive it the first time it's requested // + ////////////////////////////////////////////////////////////////// + private Boolean isMany = null; + /******************************************************************************* @@ -46,6 +59,71 @@ public class ExposedJoin + /******************************************************************************* + ** + *******************************************************************************/ + public Boolean getIsMany() + { + if(isMany == null) + { + if(CollectionUtils.nullSafeHasContents(joinPath)) + { + try + { + QInstance qInstance = QContext.getQInstance(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // loop backward through the joinPath, starting at the join table (since we don't know the table that this exposedJoin is attached to!) // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + String currentTable = joinTable; + for(int i = joinPath.size() - 1; i >= 0; i--) + { + String joinName = joinPath.get(i); + QJoinMetaData join = qInstance.getJoin(joinName); + if(join.getRightTable().equals(currentTable)) + { + currentTable = join.getLeftTable(); + if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_MANY)) + { + isMany = true; + break; + } + } + else if(join.getLeftTable().equals(currentTable)) + { + currentTable = join.getRightTable(); + if(join.getType().equals(JoinType.MANY_TO_ONE) || join.getType().equals(JoinType.MANY_TO_MANY)) + { + isMany = true; + break; + } + } + else + { + throw (new IllegalStateException("Current join table [" + currentTable + "] in path traversal was not found at element [" + joinName + "]")); + } + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we successfully got through the loop, and never found a reason to mark this join as "many", then it must not be, so set isMany to false // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(isMany == null) + { + isMany = false; + } + } + catch(Exception e) + { + LOG.warn("Error deriving if ExposedJoin through [" + joinPath + "] to [" + joinTable + "] isMany", e); + } + } + } + + return (isMany); + } + + + /******************************************************************************* ** Setter for label *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraphTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraphTest.java deleted file mode 100644 index 344a8c3b..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraphTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2023. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.actions.metadata; - - -import java.util.Set; -import com.kingsrook.qqq.backend.core.BaseTest; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import org.junit.jupiter.api.Test; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class JoinGraphTest extends BaseTest -{ - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void test() - { - JoinGraph joinGraph = new JoinGraph(QContext.getQInstance()); - for(QTableMetaData table : QContext.getQInstance().getTables().values()) - { - Set joins = joinGraph.getJoins(table.getName()); - if(joins.isEmpty()) - { - System.out.println(table.getName() + " has no joins"); - } - else - { - System.out.println(table.getName() + " joins: " + joins); - } - } - } - -} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java index ce9ffe45..4deb3559 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java @@ -32,7 +32,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; @@ -41,6 +45,7 @@ import org.junit.jupiter.api.Test; import static com.kingsrook.qqq.backend.core.utils.TestUtils.APP_NAME_GREETINGS; import static com.kingsrook.qqq.backend.core.utils.TestUtils.APP_NAME_MISCELLANEOUS; import static com.kingsrook.qqq.backend.core.utils.TestUtils.APP_NAME_PEOPLE; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -304,4 +309,131 @@ class QInstanceEnricherTest extends BaseTest } } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testExposedJoinPaths() + { + ////////////////////////// + // no join path => fail // + ////////////////////////// + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin().withJoinTable("B"))); + qInstance.addTable(newTable("B", "id", "aId")); + assertThatThrownBy(() -> new QInstanceEnricher(qInstance).enrich()) + .rootCause() + .hasMessageContaining("Could not infer a joinPath for table [A], exposedJoin to [B]") + .hasMessageContaining("No join connections between these tables exist in this instance."); + } + + ///////////////////////////////// + // multiple join paths => fail // + ///////////////////////////////// + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin().withJoinTable("B"))); + qInstance.addTable(newTable("B", "id", "aId1", "aId2")); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB1").withJoinOn(new JoinOn("id", "aId1")).withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB2").withJoinOn(new JoinOn("id", "aId2")).withType(JoinType.ONE_TO_ONE)); + assertThatThrownBy(() -> new QInstanceEnricher(qInstance).enrich()) + .rootCause() + .hasMessageContaining("Could not infer a joinPath for table [A], exposedJoin to [B]") + .hasMessageContaining("2 join connections exist") + .hasMessageContaining("\nAB1\n") + .hasMessageContaining("\nAB2."); + + //////////////////////////////////////////// + // but if you specify a path, you're good // + //////////////////////////////////////////// + qInstance.getTable("A").getExposedJoins().get(0).setJoinPath(List.of("AB2")); + new QInstanceEnricher(qInstance).enrich(); + assertEquals("B", qInstance.getTable("A").getExposedJoins().get(0).getLabel()); + } + + ///////////////////////////////// + // multiple join paths => fail // + ///////////////////////////////// + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin().withJoinTable("C"))); + qInstance.addTable(newTable("B", "id", "aId")); + qInstance.addTable(newTable("C", "id", "bId", "aId")); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB").withJoinOn(new JoinOn("id", "aId")).withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withLeftTable("B").withRightTable("C").withName("BC").withJoinOn(new JoinOn("id", "bId")).withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("C").withName("AC").withJoinOn(new JoinOn("id", "aId")).withType(JoinType.ONE_TO_ONE)); + assertThatThrownBy(() -> new QInstanceEnricher(qInstance).enrich()) + .rootCause() + .hasMessageContaining("Could not infer a joinPath for table [A], exposedJoin to [C]") + .hasMessageContaining("2 join connections exist") + .hasMessageContaining("\nAB, BC\n") + .hasMessageContaining("\nAC."); + + //////////////////////////////////////////// + // but if you specify a path, you're good // + //////////////////////////////////////////// + qInstance.getTable("A").getExposedJoins().get(0).setJoinPath(List.of("AB", "BC")); + new QInstanceEnricher(qInstance).enrich(); + assertEquals("C", qInstance.getTable("A").getExposedJoins().get(0).getLabel()); + } + + ////////////////////////////////////////////////////////////////////////////////////// + // even if you specify a bogus path, Enricher doesn't care - see validator to care. // + ////////////////////////////////////////////////////////////////////////////////////// + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin().withJoinTable("B").withJoinPath(List.of("not-a-join")))); + qInstance.addTable(newTable("B", "id", "aId")); + new QInstanceEnricher(qInstance).enrich(); + } + + //////////////////////////////////// + // one join path => great success // + //////////////////////////////////// + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.addTable(newTable("A", "id") + .withExposedJoin(new ExposedJoin().withJoinTable("B")) + .withExposedJoin(new ExposedJoin().withJoinTable("C"))); + qInstance.addTable(newTable("B", "id", "aId")); + qInstance.addTable(newTable("C", "id", "bId")); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB").withJoinOn(new JoinOn("id", "aId")).withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withLeftTable("B").withRightTable("C").withName("BC").withJoinOn(new JoinOn("id", "bId")).withType(JoinType.ONE_TO_ONE)); + + new QInstanceEnricher(qInstance).enrich(); + + ExposedJoin exposedJoinAB = qInstance.getTable("A").getExposedJoins().get(0); + assertEquals("B", exposedJoinAB.getLabel()); + assertEquals(List.of("AB"), exposedJoinAB.getJoinPath()); + + ExposedJoin exposedJoinAC = qInstance.getTable("A").getExposedJoins().get(1); + assertEquals("C", exposedJoinAC.getLabel()); + assertEquals(List.of("AB", "BC"), exposedJoinAC.getJoinPath()); + } + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData newTable(String tableName, String... fieldNames) + { + QTableMetaData tableMetaData = new QTableMetaData() + .withName(tableName) + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField(fieldNames[0]); + + for(String fieldName : fieldNames) + { + tableMetaData.addField(new QFieldMetaData(fieldName, QFieldType.INTEGER)); + } + + return (tableMetaData); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 361343cf..e6e50a68 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -50,6 +50,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; @@ -65,6 +68,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; @@ -1671,6 +1675,83 @@ class QInstanceValidatorTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testExposedJoinPaths() + { + assertValidationFailureReasons(qInstance -> qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin())), + "Table A has an exposedJoin that is missing a joinTable name", + "Table A exposedJoin [missingJoinTableName] is missing a label"); + + assertValidationFailureReasons(qInstance -> qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin().withJoinTable("B"))), + "Table A exposedJoin B is referencing an unrecognized table", + "Table A exposedJoin B is missing a label"); + + assertValidationFailureReasons(qInstance -> + { + qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin().withJoinTable("B").withLabel("B").withJoinPath(List.of("notAJoin")))); + qInstance.addTable(newTable("B", "id", "aId")); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB").withType(JoinType.ONE_TO_ONE).withJoinOn(new JoinOn("id", "aId"))); + }, + "does not match a valid join connection in the instance"); + + assertValidationFailureReasons(qInstance -> + { + qInstance.addTable(newTable("A", "id") + .withExposedJoin(new ExposedJoin().withJoinTable("B").withLabel("foo").withJoinPath(List.of("AB"))) + .withExposedJoin(new ExposedJoin().withJoinTable("C").withLabel("foo").withJoinPath(List.of("AC"))) + ); + qInstance.addTable(newTable("B", "id", "aId")); + qInstance.addTable(newTable("C", "id", "aId")); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB").withType(JoinType.ONE_TO_ONE).withJoinOn(new JoinOn("id", "aId"))); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("C").withName("AC").withType(JoinType.ONE_TO_ONE).withJoinOn(new JoinOn("id", "aId"))); + }, + "more than one join labeled: foo"); + + assertValidationFailureReasons(qInstance -> + { + qInstance.addTable(newTable("A", "id") + .withExposedJoin(new ExposedJoin().withJoinTable("B").withLabel("B1").withJoinPath(List.of("AB"))) + .withExposedJoin(new ExposedJoin().withJoinTable("B").withLabel("B2").withJoinPath(List.of("AB"))) + ); + qInstance.addTable(newTable("B", "id", "aId")); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB").withType(JoinType.ONE_TO_ONE).withJoinOn(new JoinOn("id", "aId"))); + }, + "than one join with the joinPath: [AB]"); + + assertValidationSuccess(qInstance -> + { + qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin().withJoinTable("B").withLabel("B").withJoinPath(List.of("AB")))); + qInstance.addTable(newTable("B", "id", "aId")); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB").withType(JoinType.ONE_TO_ONE).withJoinOn(new JoinOn("id", "aId"))); + }); + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData newTable(String tableName, String... fieldNames) + { + QTableMetaData tableMetaData = new QTableMetaData() + .withName(tableName) + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField(fieldNames[0]); + + for(String fieldName : fieldNames) + { + tableMetaData.addField(new QFieldMetaData(fieldName, QFieldType.INTEGER)); + } + + return (tableMetaData); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoinTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoinTest.java new file mode 100644 index 00000000..def4624d --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoinTest.java @@ -0,0 +1,137 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for ExposedJoin + *******************************************************************************/ +class ExposedJoinTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testIsManyOneToOne() + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addTable(new QTableMetaData().withName("A").withExposedJoin(new ExposedJoin().withJoinTable("B").withJoinPath(List.of("AB")))); + qInstance.addTable(new QTableMetaData().withName("B").withExposedJoin(new ExposedJoin().withJoinTable("A").withJoinPath(List.of("AB")))); + qInstance.addJoin(new QJoinMetaData().withName("AB").withLeftTable("A").withRightTable("B").withType(JoinType.ONE_TO_ONE)); + + assertFalse(qInstance.getTable("A").getExposedJoins().get(0).getIsMany()); + assertFalse(qInstance.getTable("B").getExposedJoins().get(0).getIsMany()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testIsManyOneToMany() + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addTable(new QTableMetaData().withName("A").withExposedJoin(new ExposedJoin().withJoinTable("B").withJoinPath(List.of("AB")))); + qInstance.addTable(new QTableMetaData().withName("B").withExposedJoin(new ExposedJoin().withJoinTable("A").withJoinPath(List.of("AB")))); + qInstance.addJoin(new QJoinMetaData().withName("AB").withLeftTable("A").withRightTable("B").withType(JoinType.ONE_TO_MANY)); + + assertTrue(qInstance.getTable("A").getExposedJoins().get(0).getIsMany()); + assertFalse(qInstance.getTable("B").getExposedJoins().get(0).getIsMany()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testIsManyManyToOne() + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addTable(new QTableMetaData().withName("A").withExposedJoin(new ExposedJoin().withJoinTable("B").withJoinPath(List.of("AB")))); + qInstance.addTable(new QTableMetaData().withName("B").withExposedJoin(new ExposedJoin().withJoinTable("A").withJoinPath(List.of("AB")))); + qInstance.addJoin(new QJoinMetaData().withName("AB").withLeftTable("A").withRightTable("B").withType(JoinType.MANY_TO_ONE)); + + assertFalse(qInstance.getTable("A").getExposedJoins().get(0).getIsMany()); + assertTrue(qInstance.getTable("B").getExposedJoins().get(0).getIsMany()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testIsManyOneToOneThroughChain() + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addTable(new QTableMetaData().withName("A").withExposedJoin(new ExposedJoin().withJoinTable("G").withJoinPath(List.of("AB", "BC", "CD", "DE", "EF", "FG")))); + qInstance.addTable(new QTableMetaData().withName("B").withExposedJoin(new ExposedJoin().withJoinTable("G").withJoinPath(List.of("BC", "CD", "DE", "EF", "FG")))); + qInstance.addJoin(new QJoinMetaData().withName("AB").withLeftTable("A").withRightTable("B").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("BC").withLeftTable("B").withRightTable("C").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("CD").withLeftTable("C").withRightTable("D").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("DE").withLeftTable("D").withRightTable("E").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("EF").withLeftTable("E").withRightTable("F").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("FG").withLeftTable("F").withRightTable("G").withType(JoinType.ONE_TO_ONE)); + + assertFalse(qInstance.getTable("A").getExposedJoins().get(0).getIsMany()); + assertFalse(qInstance.getTable("B").getExposedJoins().get(0).getIsMany()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testIsManyOneToManyThroughChain() + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addTable(new QTableMetaData().withName("A").withExposedJoin(new ExposedJoin().withJoinTable("G").withJoinPath(List.of("AB", "BC", "CD", "DE", "EF", "FG")))); + qInstance.addTable(new QTableMetaData().withName("B").withExposedJoin(new ExposedJoin().withJoinTable("E").withJoinPath(List.of("BC", "CD", "DE")))); + qInstance.addTable(new QTableMetaData().withName("F").withExposedJoin(new ExposedJoin().withJoinTable("C").withJoinPath(List.of("FG", "EF", "DE", "CD")))); + qInstance.addJoin(new QJoinMetaData().withName("AB").withLeftTable("A").withRightTable("B").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("BC").withLeftTable("B").withRightTable("C").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("CD").withLeftTable("C").withRightTable("D").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("DE").withLeftTable("D").withRightTable("E").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("EF").withLeftTable("E").withRightTable("F").withType(JoinType.ONE_TO_MANY)); + qInstance.addJoin(new QJoinMetaData().withName("FG").withLeftTable("F").withRightTable("G").withType(JoinType.ONE_TO_ONE)); + + assertTrue(qInstance.getTable("A").getExposedJoins().get(0).getIsMany()); + assertFalse(qInstance.getTable("B").getExposedJoins().get(0).getIsMany()); + assertFalse(qInstance.getTable("F").getExposedJoins().get(0).getIsMany()); + } + +} \ No newline at end of file From 1407f3c63cc3a871e3b1563975fd70521dcdbd62 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Apr 2023 12:12:41 -0500 Subject: [PATCH 06/15] Add count distinct option to count action --- .../actions/tables/count/CountInput.java | 34 +++++++++++++- .../actions/tables/count/CountOutput.java | 44 +++++++++++++++++++ .../rdbms/actions/RDBMSCountAction.java | 11 +++++ .../javalin/QJavalinImplementation.java | 6 +++ .../qqq/backend/javalin/QJavalinUtils.java | 18 ++++++++ 5 files changed, 112 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountInput.java index 09ff116c..1a4863e5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountInput.java @@ -37,7 +37,8 @@ public class CountInput extends AbstractTableActionInput { private QQueryFilter filter; - private List queryJoins = null; + private List queryJoins = null; + private Boolean includeDistinctCount = false; @@ -120,4 +121,35 @@ public class CountInput extends AbstractTableActionInput return (this); } + + + /******************************************************************************* + ** Getter for includeDistinctCount + *******************************************************************************/ + public Boolean getIncludeDistinctCount() + { + return (this.includeDistinctCount); + } + + + + /******************************************************************************* + ** Setter for includeDistinctCount + *******************************************************************************/ + public void setIncludeDistinctCount(Boolean includeDistinctCount) + { + this.includeDistinctCount = includeDistinctCount; + } + + + + /******************************************************************************* + ** Fluent setter for includeDistinctCount + *******************************************************************************/ + public CountInput withIncludeDistinctCount(Boolean includeDistinctCount) + { + this.includeDistinctCount = includeDistinctCount; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountOutput.java index d3703cfa..0484a175 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountOutput.java @@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; public class CountOutput extends AbstractActionOutput { private Integer count; + private Integer distinctCount; @@ -52,4 +53,47 @@ public class CountOutput extends AbstractActionOutput { this.count = count; } + + + + /******************************************************************************* + ** Getter for distinctCount + *******************************************************************************/ + public Integer getDistinctCount() + { + return (this.distinctCount); + } + + + + /******************************************************************************* + ** Setter for distinctCount + *******************************************************************************/ + public void setDistinctCount(Integer distinctCount) + { + this.distinctCount = distinctCount; + } + + + + /******************************************************************************* + ** Fluent setter for distinctCount + *******************************************************************************/ + public CountOutput withDistinctCount(Integer distinctCount) + { + this.distinctCount = distinctCount; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for count + *******************************************************************************/ + public CountOutput withCount(Integer count) + { + this.count = count; + return (this); + } + } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java index dad1067f..e713c1b5 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import org.apache.commons.lang.BooleanUtils; /******************************************************************************* @@ -63,6 +64,11 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf String primaryKeyColumn = escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(fieldAndTableNameOrAlias.field().getName()); String clausePrefix = (requiresDistinct) ? "SELECT COUNT(DISTINCT (" + primaryKeyColumn + "))" : "SELECT COUNT(*)"; + if(BooleanUtils.isTrue(countInput.getIncludeDistinctCount())) + { + clausePrefix = "SELECT COUNT(DISTINCT (" + primaryKeyColumn + ")) AS distinct_count, COUNT(*)"; + } + String sql = clausePrefix + " AS record_count FROM " + makeFromClause(countInput.getInstance(), table.getName(), joinsContext); @@ -81,6 +87,11 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf if(resultSet.next()) { rs.setCount(resultSet.getInt("record_count")); + + if(BooleanUtils.isTrue(countInput.getIncludeDistinctCount())) + { + rs.setDistinctCount(resultSet.getInt("distinct_count")); + } } }), params); diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 62c88550..a8eaae97 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -788,6 +788,7 @@ public class QJavalinImplementation } countInput.setQueryJoins(processQueryJoinsParam(context)); + countInput.setIncludeDistinctCount(QJavalinUtils.queryParamIsTrue(context, "includeDistinct")); CountAction countAction = new CountAction(); CountOutput countOutput = countAction.execute(countInput); @@ -861,6 +862,11 @@ public class QJavalinImplementation QueryAction queryAction = new QueryAction(); QueryOutput queryOutput = queryAction.execute(queryInput); + int rowIndex = 0; + for(QRecord record : queryOutput.getRecords()) + { + record.setValue("__qRowIndex", rowIndex++); + } QJavalinAccessLogger.logEndSuccess(logPair("recordCount", queryOutput.getRecords().size()), logPairIfSlow("filter", filter, SLOW_LOG_THRESHOLD_MS)); context.result(JsonUtils.toJson(queryOutput)); diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinUtils.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinUtils.java index d8fa1ebe..fb1a4681 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinUtils.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinUtils.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.javalin; +import java.util.Objects; import com.kingsrook.qqq.backend.core.exceptions.QValueException; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -52,6 +53,23 @@ public class QJavalinUtils + /******************************************************************************* + ** Returns true iff context has a valid query parameter by the given name, with + ** a value of "true". + *******************************************************************************/ + public static boolean queryParamIsTrue(Context context, String name) throws QValueException + { + String value = context.queryParam(name); + if(Objects.equals(value, "true")) + { + return (true); + } + + return (false); + } + + + /******************************************************************************* ** Returns Integer if context has a valid int form parameter by the given name, ** Returns null if no param (or empty value). From 475deee99308d38b6fb8d2095512fe63393683ec Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Apr 2023 12:12:54 -0500 Subject: [PATCH 07/15] Upgrade javalin to 5.4.2 --- qqq-middleware-javalin/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-middleware-javalin/pom.xml b/qqq-middleware-javalin/pom.xml index d225ee0f..78e38af5 100644 --- a/qqq-middleware-javalin/pom.xml +++ b/qqq-middleware-javalin/pom.xml @@ -60,7 +60,7 @@ io.javalin javalin - 5.1.4 + 5.4.2 com.konghq From de77f902ac500a70314870bc65f419ab6de0793f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Apr 2023 12:53:39 -0500 Subject: [PATCH 08/15] Removed wip code, not meant to be commited --- .../core/actions/metadata/MetaDataAction.java | 149 ------------------ 1 file changed, 149 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java index f0bde491..ef571e45 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java @@ -23,21 +23,16 @@ package com.kingsrook.qqq.backend.core.actions.metadata; import java.util.ArrayList; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; -import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNode; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendAppMetaData; @@ -45,7 +40,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendProcessMe import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendWidgetMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; @@ -221,149 +215,6 @@ public class MetaDataAction return metaDataOutput; } - ////////////////////////////////////// start v1 ////////////////////////////////////// - - - - private record JoinedTable(String joinedTableName, List joinPath) - { - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void addJoinedTablesToTables(Map tables) - { - for(QFrontendTableMetaData table : tables.values()) - { - List joinedTables = new ArrayList<>(); - addJoinedTablesToTable(tables, table, joinedTables, new ArrayList<>()); - - if(joinedTables.size() > 0) - { - System.out.println("For [" + table.getName() + "] we have:\n " + joinedTables.stream().map(String::valueOf).collect(Collectors.joining("\n ")) + "\n"); - } - else - { - System.out.println("No joins for [" + table.getName() + "]\n"); - } - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void addJoinedTablesToTable(Map tables, QFrontendTableMetaData table, List joinedTables, List joinPath) - { - QInstance qInstance = QContext.getQInstance(); - for(QJoinMetaData join : qInstance.getJoins().values()) - { - if(join.getLeftTable().equals(table.getName())) - { - String joinName = join.getName(); - JoinedTable joinedTable = new JoinedTable(join.getRightTable(), joinPath); - System.out.println("Adding to [" + table.getName() + "]: " + joinedTable); - joinedTables.add(joinedTable); - - ArrayList subJoinPath = new ArrayList<>(joinPath); - subJoinPath.add(joinName); - addJoinedTablesToTable(tables, tables.get(join.getRightTable()), joinedTables, subJoinPath); - } - if(join.getRightTable().equals(table.getName())) - { - String joinName = join.getName() + ".flipped"; - JoinedTable joinedTable = new JoinedTable(join.getLeftTable(), joinPath); - System.out.println("Adding to [" + table.getName() + "]: " + joinedTable); - joinedTables.add(joinedTable); - - ArrayList subJoinPath = new ArrayList<>(joinPath); - subJoinPath.add(joinName); - addJoinedTablesToTable(tables, tables.get(join.getLeftTable()), joinedTables, subJoinPath); - } - } - } - - ////////////////////////////////////// end v1 ////////////////////////////////////// - - ////////////////////////////////////// start v0 ////////////////////////////////////// - - - - private record Something(String joinName, List joinPath) - { - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void addJoinsToTables(Map tables) - { - for(QFrontendTableMetaData table : tables.values()) - { - List something = new ArrayList<>(); - addJoinsToTable(tables, table, something, new ArrayList<>(), new HashSet<>()); - if(something.size() > 0) - { - System.out.println("For [" + table.getName() + "] we have:\n " + something.stream().map(String::valueOf).collect(Collectors.joining("\n ")) + "\n"); - } - else - { - System.out.println("No joins for [" + table.getName() + "]\n"); - } - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void addJoinsToTable(Map tables, QFrontendTableMetaData table, List something, List joinPath, Set usedJoins) - { - QInstance qInstance = QContext.getQInstance(); - for(QJoinMetaData join : qInstance.getJoins().values()) - { - if(join.getLeftTable().equals(table.getName())) - { - String joinName = join.getName(); - if(!usedJoins.contains(joinName)) - { - usedJoins.add(joinName); - something.add(new Something(joinName, joinPath)); - - ArrayList subJoinPath = new ArrayList<>(joinPath); - subJoinPath.add(joinName); - - QFrontendTableMetaData rightTable = tables.get(join.getRightTable()); - addJoinsToTable(tables, rightTable, something, subJoinPath, usedJoins); - } - } - else if(join.getRightTable().equals(table.getName())) - { - String joinName = join.getName() + ".flipped"; - if(!usedJoins.contains(joinName)) - { - usedJoins.add(joinName); - something.add(new Something(joinName, joinPath)); - - ArrayList subJoinPath = new ArrayList<>(joinPath); - subJoinPath.add(joinName); - - QFrontendTableMetaData leftTable = tables.get(join.getLeftTable()); - addJoinsToTable(tables, leftTable, something, subJoinPath, usedJoins); - } - } - } - } - - ////////////////////////////////////// end v0 ////////////////////////////////////// - /******************************************************************************* From caf9f102f65938638f3a30240f3aaee432dae011 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Apr 2023 12:54:42 -0500 Subject: [PATCH 09/15] Moved stuff so jacoco reporting happens before failures, i think. Moved untested class reporting into pom, out of circleci --- .circleci/config.yml | 6 +-- pom.xml | 103 +++++++++++++++++++++++++------------------ 2 files changed, 60 insertions(+), 49 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3e2ff6f9..01b5b8c9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,6 +26,7 @@ commands: sudo rm /etc/alternatives/java sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java - run: + ## used by jacoco uncovered class reporting in pom.xml name: Install html2text command: | sudo apt-get update @@ -65,11 +66,6 @@ commands: when: always - store_test_results: path: ~/test-results - - run: - name: Find Un-tested Classes - command: | - set +o pipefail && for i in */target/site/jacoco/*/index.html; do html2text -width 500 -nobs $i | sed '1,/^Total/d;' | grep -v Created | sed 's/ \+/ /g' | sed 's/ [[:digit:]]$//' | grep -v 0$ | cut -d' ' -f1; done - when: always - save_cache: paths: - ~/.m2 diff --git a/pom.xml b/pom.xml index 2bc0ecf0..4623b96d 100644 --- a/pom.xml +++ b/pom.xml @@ -206,6 +206,63 @@ true + + exec-maven-plugin + org.codehaus.mojo + 3.0.0 + + + test-coverage-summary + verify + + exec + + + sh + + -c + + /dev/null 2>&1 +if [ "$?" == "0" ]; then + echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers + xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values + paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}' + rm /tmp/$$.headers /tmp/$$.values +else + echo "xpath is not installed. Jacoco coverage summary will not be produced here..."; +fi + +which xpath > /dev/null 2>&1 +if [ "$?" == "0" ]; then + echo "Untested classes, per Jacoco:" + echo "-----------------------------" + for i in target/site/jacoco/*/index.html; do + html2text -width 500 -nobs $i | sed '1,/^Total/d;' | grep -v Created | sed 's/ \+/ /g' | sed 's/ [[:digit:]]$//' | grep -v 0$ | cut -d' ' -f1; + done; + echo +else + echo "html2text is not installed. Untested classes from Jacoco will not be printed here..."; +fi + + ]]> + + + + + + org.jacoco jacoco-maven-plugin @@ -249,56 +306,14 @@ post-unit-test - verify + + post-integration-test report - - exec-maven-plugin - org.codehaus.mojo - 3.0.0 - - - test-coverage-summary - verify - - exec - - - sh - - -c - - /dev/null 2>&1 -if [ "$?" == "0" ]; then - echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers - xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values - paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}' - rm /tmp/$$.headers /tmp/$$.values -else - echo "xpath is not installed. Jacoco coverage summary will not be produced here.."; -fi - ]]> - - - - - - From 04a8fa94f9bcbdcdb0f8c461ab9d36cf2c294dfd Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Apr 2023 10:18:44 -0500 Subject: [PATCH 10/15] Move skip & limit out of QueryInput, into QQueryFilter... --- .../widgets/ChildRecordListRenderer.java | 2 +- .../scripts/StoreAssociatedScriptAction.java | 3 +- .../SearchPossibleValueSourceAction.java | 2 +- .../actions/tables/query/QQueryFilter.java | 69 ++++++++++++ .../actions/tables/query/QueryInput.java | 102 ++++++------------ .../dashboard/nocode/WidgetQueryField.java | 5 +- .../enumeration/EnumerationQueryAction.java | 2 +- .../implementations/mock/MockQueryAction.java | 5 +- .../utils/BackendQueryFilterUtils.java | 82 +++++++------- .../ExtractViaQueryStep.java | 43 +++++++- .../StoreScriptRevisionProcessStep.java | 3 +- .../processes/utils/GeneralProcessUtils.java | 3 +- .../EnumerationQueryActionTest.java | 21 ++-- .../memory/MemoryBackendModuleTest.java | 7 +- .../module/api/actions/BaseAPIActionUtil.java | 17 +-- .../rdbms/actions/RDBMSQueryAction.java | 10 +- .../qqq/api/javalin/QJavalinApiHandler.java | 7 +- .../javalin/QJavalinImplementation.java | 19 ++-- .../javalin/QJavalinScriptsHandler.java | 3 +- .../picocli/QPicoCliImplementation.java | 7 +- .../qqq/slack/QSlackImplementation.java | 3 +- 21 files changed, 244 insertions(+), 171 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java index 5a068d8d..92cacd2f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java @@ -194,13 +194,13 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(record.getValue(joinOn.getLeftField())))); } filter.setOrderBys(join.getOrderBys()); + filter.setLimit(maxRows); QueryInput queryInput = new QueryInput(); queryInput.setTableName(join.getRightTable()); queryInput.setShouldTranslatePossibleValues(true); queryInput.setShouldGenerateDisplayValues(true); queryInput.setFilter(filter); - queryInput.setLimit(maxRows); QueryOutput queryOutput = new QueryAction().execute(queryInput); QTableMetaData table = input.getInstance().getTable(join.getRightTable()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java index 1d2d8848..c9cc5100 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java @@ -156,8 +156,7 @@ public class StoreAssociatedScriptAction queryInput.setFilter(new QQueryFilter() .withCriteria(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, List.of(script.getValue("id")))) .withOrderBy(new QFilterOrderBy("sequenceNo", false)) - ); - queryInput.setLimit(1); + .withLimit(1)); QueryOutput queryOutput = new QueryAction().execute(queryInput); if(!queryOutput.getRecords().isEmpty()) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java index c034e1f7..8a25136c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java @@ -247,7 +247,7 @@ public class SearchPossibleValueSourceAction queryFilter.setOrderBys(possibleValueSource.getOrderByFields()); // todo - skip & limit as params - queryInput.setLimit(250); + queryFilter.setLimit(250); /////////////////////////////////////////////////////////////////////////////////////////////////////// // if given a default filter, make it the 'top level' filter and the one we just created a subfilter // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java index f89d23e4..6ce122bb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java @@ -47,6 +47,13 @@ public class QQueryFilter implements Serializable, Cloneable private BooleanOperator booleanOperator = BooleanOperator.AND; private List subFilters = new ArrayList<>(); + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // skip & limit are meant to only apply to QueryAction (at least at the initial time they are added here) // + // e.g., they are ignored in CountAction, AggregateAction, etc, where their meanings may be less obvious // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + private Integer skip; + private Integer limit; + /******************************************************************************* @@ -398,4 +405,66 @@ public class QQueryFilter implements Serializable, Cloneable } } + + + /******************************************************************************* + ** Getter for skip + *******************************************************************************/ + public Integer getSkip() + { + return (this.skip); + } + + + + /******************************************************************************* + ** Setter for skip + *******************************************************************************/ + public void setSkip(Integer skip) + { + this.skip = skip; + } + + + + /******************************************************************************* + ** Fluent setter for skip + *******************************************************************************/ + public QQueryFilter withSkip(Integer skip) + { + this.skip = skip; + return (this); + } + + + + /******************************************************************************* + ** Getter for limit + *******************************************************************************/ + public Integer getLimit() + { + return (this.limit); + } + + + + /******************************************************************************* + ** Setter for limit + *******************************************************************************/ + public void setLimit(Integer limit) + { + this.limit = limit; + } + + + + /******************************************************************************* + ** Fluent setter for limit + *******************************************************************************/ + public QQueryFilter withLimit(Integer limit) + { + this.limit = limit; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java index 3a49001e..d1335dbe 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java @@ -39,8 +39,6 @@ public class QueryInput extends AbstractTableActionInput { private QBackendTransaction transaction; private QQueryFilter filter; - private Integer skip; - private Integer limit; private RecordPipe recordPipe; @@ -55,7 +53,8 @@ public class QueryInput extends AbstractTableActionInput ///////////////////////////////////////////////////////////////////////////////////////// private Set fieldsToTranslatePossibleValues; - private List queryJoins = null; + private List queryJoins = null; + private boolean selectDistinct = false; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if you say you want to includeAssociations, you can limit which ones by passing them in associationNamesToInclude. // @@ -98,50 +97,6 @@ public class QueryInput extends AbstractTableActionInput - /******************************************************************************* - ** Getter for skip - ** - *******************************************************************************/ - public Integer getSkip() - { - return skip; - } - - - - /******************************************************************************* - ** Setter for skip - ** - *******************************************************************************/ - public void setSkip(Integer skip) - { - this.skip = skip; - } - - - - /******************************************************************************* - ** Getter for limit - ** - *******************************************************************************/ - public Integer getLimit() - { - return limit; - } - - - - /******************************************************************************* - ** Setter for limit - ** - *******************************************************************************/ - public void setLimit(Integer limit) - { - this.limit = limit; - } - - - /******************************************************************************* ** Getter for recordPipe ** @@ -359,28 +314,6 @@ public class QueryInput extends AbstractTableActionInput - /******************************************************************************* - ** Fluent setter for skip - *******************************************************************************/ - public QueryInput withSkip(Integer skip) - { - this.skip = skip; - return (this); - } - - - - /******************************************************************************* - ** Fluent setter for limit - *******************************************************************************/ - public QueryInput withLimit(Integer limit) - { - this.limit = limit; - return (this); - } - - - /******************************************************************************* ** Fluent setter for recordPipe *******************************************************************************/ @@ -497,4 +430,35 @@ public class QueryInput extends AbstractTableActionInput return (this); } + + + /******************************************************************************* + ** Getter for selectDistinct + *******************************************************************************/ + public boolean getSelectDistinct() + { + return (this.selectDistinct); + } + + + + /******************************************************************************* + ** Setter for selectDistinct + *******************************************************************************/ + public void setSelectDistinct(boolean selectDistinct) + { + this.selectDistinct = selectDistinct; + } + + + + /******************************************************************************* + ** Fluent setter for selectDistinct + *******************************************************************************/ + public QueryInput withSelectDistinct(boolean selectDistinct) + { + this.selectDistinct = selectDistinct; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/WidgetQueryField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/WidgetQueryField.java index f5997b6d..197ce195 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/WidgetQueryField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/WidgetQueryField.java @@ -62,8 +62,9 @@ public class WidgetQueryField extends AbstractWidgetValueSourceWithFilter { QueryInput queryInput = new QueryInput(); queryInput.setTableName(tableName); - queryInput.setFilter(getEffectiveFilter(input)); - queryInput.setLimit(1); + QQueryFilter effectiveFilter = getEffectiveFilter(input); + queryInput.setFilter(effectiveFilter); + effectiveFilter.setLimit(1); QueryOutput queryOutput = new QueryAction().execute(queryInput); if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords())) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryAction.java index 039e0123..c04d1dd1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryAction.java @@ -69,7 +69,7 @@ public class EnumerationQueryAction implements QueryInterface } BackendQueryFilterUtils.sortRecordList(queryInput.getFilter(), recordList); - recordList = BackendQueryFilterUtils.applySkipAndLimit(queryInput, recordList); + recordList = BackendQueryFilterUtils.applySkipAndLimit(queryInput.getFilter(), recordList); QueryOutput queryOutput = new QueryOutput(queryInput); queryOutput.addRecords(recordList); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java index 0d4b918a..d66791f2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java @@ -27,7 +27,6 @@ import java.math.BigDecimal; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.Month; -import java.util.Objects; import java.util.UUID; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -36,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; /******************************************************************************* @@ -59,7 +59,8 @@ public class MockQueryAction implements QueryInterface QueryOutput queryOutput = new QueryOutput(queryInput); - int rows = Objects.requireNonNullElse(queryInput.getLimit(), 1); + @SuppressWarnings("UnnecessaryUnboxing") // force an un-boxing, to force an NPE if it's null, to get to the "else 1" + int rows = ObjectUtils.tryElse(() -> queryInput.getFilter().getLimit().intValue(), 1); for(int i = 0; i < rows; i++) { QRecord record = new QRecord(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java index abf691d8..e05d0bbd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java @@ -33,7 +33,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -129,42 +128,42 @@ public class BackendQueryFilterUtils public static boolean doesCriteriaMatch(QFilterCriteria criterion, String fieldName, Serializable value) { boolean criterionMatches = switch(criterion.getOperator()) + { + case EQUALS -> testEquals(criterion, value); + case NOT_EQUALS -> !testEquals(criterion, value); + case IN -> testIn(criterion, value); + case NOT_IN -> !testIn(criterion, value); + case IS_BLANK -> testBlank(criterion, value); + case IS_NOT_BLANK -> !testBlank(criterion, value); + case CONTAINS -> testContains(criterion, fieldName, value); + case NOT_CONTAINS -> !testContains(criterion, fieldName, value); + case IS_NULL_OR_IN -> testBlank(criterion, value) || testIn(criterion, value); + case LIKE -> testLike(criterion, fieldName, value); + case NOT_LIKE -> !testLike(criterion, fieldName, value); + case STARTS_WITH -> testStartsWith(criterion, fieldName, value); + case NOT_STARTS_WITH -> !testStartsWith(criterion, fieldName, value); + case ENDS_WITH -> testEndsWith(criterion, fieldName, value); + case NOT_ENDS_WITH -> !testEndsWith(criterion, fieldName, value); + case GREATER_THAN -> testGreaterThan(criterion, value); + case GREATER_THAN_OR_EQUALS -> testGreaterThan(criterion, value) || testEquals(criterion, value); + case LESS_THAN -> !testGreaterThan(criterion, value) && !testEquals(criterion, value); + case LESS_THAN_OR_EQUALS -> !testGreaterThan(criterion, value); + case BETWEEN -> { - case EQUALS -> testEquals(criterion, value); - case NOT_EQUALS -> !testEquals(criterion, value); - case IN -> testIn(criterion, value); - case NOT_IN -> !testIn(criterion, value); - case IS_BLANK -> testBlank(criterion, value); - case IS_NOT_BLANK -> !testBlank(criterion, value); - case CONTAINS -> testContains(criterion, fieldName, value); - case NOT_CONTAINS -> !testContains(criterion, fieldName, value); - case IS_NULL_OR_IN -> testBlank(criterion, value) || testIn(criterion, value); - case LIKE -> testLike(criterion, fieldName, value); - case NOT_LIKE -> !testLike(criterion, fieldName, value); - case STARTS_WITH -> testStartsWith(criterion, fieldName, value); - case NOT_STARTS_WITH -> !testStartsWith(criterion, fieldName, value); - case ENDS_WITH -> testEndsWith(criterion, fieldName, value); - case NOT_ENDS_WITH -> !testEndsWith(criterion, fieldName, value); - case GREATER_THAN -> testGreaterThan(criterion, value); - case GREATER_THAN_OR_EQUALS -> testGreaterThan(criterion, value) || testEquals(criterion, value); - case LESS_THAN -> !testGreaterThan(criterion, value) && !testEquals(criterion, value); - case LESS_THAN_OR_EQUALS -> !testGreaterThan(criterion, value); - case BETWEEN -> - { - QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues()); - QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues())); - criteria1.getValues().remove(0); - yield (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value)); - } - case NOT_BETWEEN -> - { - QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues()); - QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues())); - criteria1.getValues().remove(0); - boolean between = (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value)); - yield !between; - } - }; + QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues()); + QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues())); + criteria1.getValues().remove(0); + yield (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value)); + } + case NOT_BETWEEN -> + { + QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues()); + QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues())); + criteria1.getValues().remove(0); + boolean between = (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value)); + yield !between; + } + }; return criterionMatches; } @@ -524,9 +523,14 @@ public class BackendQueryFilterUtils /******************************************************************************* ** Apply skip & limit attributes from queryInput to a list of records. *******************************************************************************/ - public static List applySkipAndLimit(QueryInput queryInput, List recordList) + public static List applySkipAndLimit(QQueryFilter queryFilter, List recordList) { - Integer skip = queryInput.getSkip(); + if(queryFilter == null) + { + return (recordList); + } + + Integer skip = queryFilter.getSkip(); if(skip != null && skip > 0) { if(skip < recordList.size()) @@ -539,7 +543,7 @@ public class BackendQueryFilterUtils } } - Integer limit = queryInput.getLimit(); + Integer limit = queryFilter.getLimit(); if(limit != null && limit >= 0 && limit < recordList.size()) { recordList = recordList.subList(0, limit); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java index 414b0c54..0b85c841 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java @@ -29,6 +29,7 @@ import java.util.List; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; @@ -52,6 +53,8 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; *******************************************************************************/ public class ExtractViaQueryStep extends AbstractExtractStep { + private static final QLogger LOG = QLogger.getLogger(ExtractViaQueryStep.class); + public static final String FIELD_SOURCE_TABLE = "sourceTable"; private QQueryFilter queryFilter; @@ -77,11 +80,33 @@ public class ExtractViaQueryStep extends AbstractExtractStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { + ////////////////////////////////////////////////////////////////// + // clone the filter, since we're going to edit it (set a limit) // + ////////////////////////////////////////////////////////////////// + QQueryFilter filterClone = queryFilter.clone(); + + ////////////////////////////////////////////////////////////////////////////////////////////// + // if there's a limit in the extract step (e.g., the 20-record limit on the preview screen) // + // then set that limit in the filter - UNLESS - there's already a limit in the filter for // + // a smaller number of records. // + ////////////////////////////////////////////////////////////////////////////////////////////// + if(getLimit() != null) + { + if(filterClone.getLimit() != null && filterClone.getLimit() < getLimit()) + { + LOG.trace("Using filter's limit [" + filterClone.getLimit() + "] rather than step's limit [" + getLimit() + "]"); + } + else + { + filterClone.setLimit(getLimit()); + } + } + QueryInput queryInput = new QueryInput(); queryInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE)); - queryInput.setFilter(queryFilter); + queryInput.setFilter(filterClone); + queryInput.setSelectDistinct(true); queryInput.setRecordPipe(getRecordPipe()); - queryInput.setLimit(getLimit()); queryInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback()); new QueryAction().execute(queryInput); @@ -101,8 +126,20 @@ public class ExtractViaQueryStep extends AbstractExtractStep CountInput countInput = new CountInput(); countInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE)); countInput.setFilter(queryFilter); + countInput.setIncludeDistinctCount(true); CountOutput countOutput = new CountAction().execute(countInput); - return (countOutput.getCount()); + Integer count = countOutput.getDistinctCount(); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // in case the filter we're running has a limit, but the count found more than that limit, // + // well then, just return that limit - as the process won't run on more rows than that. // + ///////////////////////////////////////////////////////////////////////////////////////////// + if(count != null & queryFilter.getLimit() != null && count > queryFilter.getLimit()) + { + count = queryFilter.getLimit(); + } + + return count; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java index d1a0fb82..b887ff9a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java @@ -85,8 +85,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep queryInput.setFilter(new QQueryFilter() .withCriteria(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, List.of(script.getValue("id")))) .withOrderBy(new QFilterOrderBy("sequenceNo", false)) - ); - queryInput.setLimit(1); + .withLimit(1)); QueryOutput queryOutput = new QueryAction().execute(queryInput); if(!queryOutput.getRecords().isEmpty()) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java index 4cde210b..98a41a4f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java @@ -209,8 +209,7 @@ public class GeneralProcessUtils QueryInput queryInput = new QueryInput(); queryInput.setTableName(tableName); - queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, fieldValue))); - queryInput.setLimit(1); + queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, fieldValue)).withLimit(1)); QueryOutput queryOutput = new QueryAction().execute(queryInput); return (queryOutput.getRecords().stream().findFirst()); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java index d1b13a98..909c9f7c 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java @@ -128,38 +128,31 @@ class EnumerationQueryActionTest extends BaseTest QueryInput queryInput = new QueryInput(); queryInput.setTableName("statesEnum"); - queryInput.setSkip(0); - queryInput.setLimit(null); + queryInput.setFilter(new QQueryFilter().withSkip(0).withLimit(null)); QueryOutput queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of("Missouri", "Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(1); - queryInput.setLimit(null); + queryInput.setFilter(new QQueryFilter().withSkip(1).withLimit(null)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of("Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(2); - queryInput.setLimit(null); + queryInput.setFilter(new QQueryFilter().withSkip(2).withLimit(null)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of(), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(null); - queryInput.setLimit(1); + queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(1)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of("Missouri"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(null); - queryInput.setLimit(2); + queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(2)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of("Missouri", "Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(null); - queryInput.setLimit(3); + queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(3)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of("Missouri", "Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(null); - queryInput.setLimit(0); + queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(0)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of(), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java index 34a25a95..23b8ab93 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java @@ -341,14 +341,13 @@ class MemoryBackendModuleTest extends BaseTest { QueryInput queryInput = new QueryInput(); queryInput.setTableName(table.getName()); - queryInput.setLimit(2); + queryInput.setFilter(new QQueryFilter().withLimit(2)); assertEquals(2, new QueryAction().execute(queryInput).getRecords().size()); - queryInput.setLimit(1); + queryInput.setFilter(new QQueryFilter().withLimit(1)); assertEquals(1, new QueryAction().execute(queryInput).getRecords().size()); - queryInput.setSkip(4); - queryInput.setLimit(3); + queryInput.setFilter(new QQueryFilter().withSkip(4).withLimit(3)); assertEquals(0, new QueryAction().execute(queryInput).getRecords().size()); } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index d8ecf17a..3f0a230c 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -228,10 +228,12 @@ public class BaseAPIActionUtil *******************************************************************************/ public QueryOutput doQuery(QTableMetaData table, QueryInput queryInput) throws QException { - QueryOutput queryOutput = new QueryOutput(queryInput); - Integer originalLimit = queryInput.getLimit(); - Integer limit = originalLimit; - Integer skip = queryInput.getSkip(); + QueryOutput queryOutput = new QueryOutput(queryInput); + QQueryFilter filter = queryInput.getFilter(); + + Integer originalLimit = filter == null ? null : filter.getLimit(); + Integer limit = originalLimit; + Integer skip = filter == null ? null : filter.getSkip(); if(limit == null) { @@ -243,10 +245,9 @@ public class BaseAPIActionUtil { try { - QQueryFilter filter = queryInput.getFilter(); - String paramString = buildQueryStringForGet(filter, limit, skip, table.getFields()); - String url = buildTableUrl(table) + paramString; - HttpGet request = new HttpGet(url); + String paramString = buildQueryStringForGet(filter, limit, skip, table.getFields()); + String url = buildTableUrl(table) + paramString; + HttpGet request = new HttpGet(url); QHttpResponse response = makeRequest(table, request); int count = processGetResponse(table, response, queryOutput); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 412dd35a..4522282f 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -83,14 +83,14 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf sql.append(" ORDER BY ").append(makeOrderByClause(table, filter.getOrderBys(), joinsContext)); } - if(queryInput.getLimit() != null) + if(filter != null && filter.getLimit() != null) { - sql.append(" LIMIT ").append(queryInput.getLimit()); + sql.append(" LIMIT ").append(filter.getLimit()); - if(queryInput.getSkip() != null) + if(filter.getSkip() != null) { // todo - other sql grammars? - sql.append(" OFFSET ").append(queryInput.getSkip()); + sql.append(" OFFSET ").append(filter.getSkip()); } } @@ -200,7 +200,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf List queryJoins = queryInput.getQueryJoins(); QTableMetaData table = instance.getTable(tableName); - boolean requiresDistinct = doesSelectClauseRequireDistinct(table); + boolean requiresDistinct = queryInput.getSelectDistinct() || doesSelectClauseRequireDistinct(table); String clausePrefix = (requiresDistinct) ? "SELECT DISTINCT " : "SELECT "; List fieldList = new ArrayList<>(table.getFields().values()); diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java index 0cb92760..72222593 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java @@ -913,6 +913,8 @@ public class QJavalinApiHandler PermissionsHelper.checkTablePermissionThrowing(queryInput, TablePermissionSubType.READ); + filter = new QQueryFilter(); + Integer pageSize = 50; if(StringUtils.hasContent(context.queryParam("pageSize"))) { @@ -947,12 +949,11 @@ public class QJavalinApiHandler badRequestMessages.add("pageNo must be greater than 0."); } - queryInput.setLimit(pageSize); - queryInput.setSkip((pageNo - 1) * pageSize); + filter.setLimit(pageSize); + filter.setSkip((pageNo - 1) * pageSize); // queryInput.setQueryJoins(processQueryJoinsParam(context)); - filter = new QQueryFilter(); if("and".equalsIgnoreCase(context.queryParam("booleanOperator"))) { filter.setBooleanOperator(QQueryFilter.BooleanOperator.AND); diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index a8eaae97..ddf3324c 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -843,8 +843,6 @@ public class QJavalinImplementation queryInput.setTableName(table); queryInput.setShouldGenerateDisplayValues(true); queryInput.setShouldTranslatePossibleValues(true); - queryInput.setSkip(QJavalinUtils.integerQueryParam(context, "skip")); - queryInput.setLimit(QJavalinUtils.integerQueryParam(context, "limit")); PermissionsHelper.checkTablePermissionThrowing(queryInput, TablePermissionSubType.READ); @@ -858,15 +856,22 @@ public class QJavalinImplementation queryInput.setFilter(JsonUtils.toObject(filter, QQueryFilter.class)); } + Integer skip = QJavalinUtils.integerQueryParam(context, "skip"); + Integer limit = QJavalinUtils.integerQueryParam(context, "limit"); + if(skip != null || limit != null) + { + if(queryInput.getFilter() == null) + { + queryInput.setFilter(new QQueryFilter()); + } + queryInput.getFilter().setSkip(skip); + queryInput.getFilter().setLimit(limit); + } + queryInput.setQueryJoins(processQueryJoinsParam(context)); QueryAction queryAction = new QueryAction(); QueryOutput queryOutput = queryAction.execute(queryInput); - int rowIndex = 0; - for(QRecord record : queryOutput.getRecords()) - { - record.setValue("__qRowIndex", rowIndex++); - } QJavalinAccessLogger.logEndSuccess(logPair("recordCount", queryOutput.getRecords().size()), logPairIfSlow("filter", filter, SLOW_LOG_THRESHOLD_MS)); context.result(JsonUtils.toJson(queryOutput)); diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinScriptsHandler.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinScriptsHandler.java index 61f84bfa..b8cfb7b2 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinScriptsHandler.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinScriptsHandler.java @@ -254,8 +254,7 @@ public class QJavalinScriptsHandler queryInput.setFilter(new QQueryFilter() .withCriteria(new QFilterCriteria("scriptRevisionId", QCriteriaOperator.EQUALS, List.of(scriptRevisionId))) .withOrderBy(new QFilterOrderBy("id", false)) - ); - queryInput.setLimit(100); + .withLimit(100)); QueryOutput queryOutput = new QueryAction().execute(queryInput); if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords())) diff --git a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java index b636d735..093a5f0d 100644 --- a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java +++ b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java @@ -531,7 +531,6 @@ public class QPicoCliImplementation { QueryInput queryInput = new QueryInput(); queryInput.setTableName(tableName); - queryInput.setSkip(subParseResult.matchedOptionValue("skip", null)); // todo - think about these (e.g., based on user's requested output format? // queryInput.setShouldGenerateDisplayValues(true); @@ -553,6 +552,8 @@ public class QPicoCliImplementation .withValues(List.of(primaryKeyValue))); queryInput.setFilter(filter); + filter.setSkip(subParseResult.matchedOptionValue("skip", null)); + QueryAction queryAction = new QueryAction(); QueryOutput queryOutput = queryAction.execute(queryInput); List records = queryOutput.getRecords(); @@ -577,9 +578,9 @@ public class QPicoCliImplementation { QueryInput queryInput = new QueryInput(); queryInput.setTableName(tableName); - queryInput.setSkip(subParseResult.matchedOptionValue("skip", null)); - queryInput.setLimit(subParseResult.matchedOptionValue("limit", null)); queryInput.setFilter(generateQueryFilter(subParseResult)); + queryInput.getFilter().setSkip(subParseResult.matchedOptionValue("skip", null)); + queryInput.getFilter().setLimit(subParseResult.matchedOptionValue("limit", null)); // todo - think about these (e.g., based on user's requested output format? // queryInput.setShouldGenerateDisplayValues(true); diff --git a/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java b/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java index 82527128..56ee591c 100644 --- a/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java +++ b/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java @@ -52,6 +52,7 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; @@ -384,7 +385,7 @@ public class QSlackImplementation try { QueryInput queryInput = new QueryInput(); - queryInput.setLimit(10); + queryInput.setFilter(new QQueryFilter().withLimit(10)); queryInput.setTableName(tableName); setupSession(context, queryInput); QueryOutput output = new QueryAction().execute(queryInput); From f7f001d430d58f2e92881e699645dbcbd47a81ac Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Apr 2023 10:18:55 -0500 Subject: [PATCH 11/15] Improve tense of okSummary --- .../bulk/edit/BulkEditTransformStep.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java index f1f9c873..b3dddc99 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java @@ -211,14 +211,11 @@ public class BulkEditTransformStep extends AbstractTransformStep @Override public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) { - if(isForResultScreen) - { - okSummary.setMessage(tableLabel + " records were edited."); - } - else - { - okSummary.setMessage(tableLabel + " records will be edited."); - } + okSummary.setSingularFutureMessage(tableLabel + " record will be edited."); + okSummary.setPluralFutureMessage(tableLabel + " records will be edited."); + okSummary.setSingularPastMessage(tableLabel + " record was edited."); + okSummary.setPluralPastMessage(tableLabel + " records were edited."); + okSummary.pickMessage(isForResultScreen); ArrayList rs = new ArrayList<>(); rs.add(okSummary); From 8094c29ec7e365b79d2e6bae01863538ecdb7c36 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Apr 2023 10:19:12 -0500 Subject: [PATCH 12/15] handle ExposedJoins in exports --- .../core/actions/reporting/ExportAction.java | 91 +++++++++++++++++-- .../memory/MemoryRecordStore.java | 28 ++++-- .../actions/reporting/ExportActionTest.java | 69 ++++++++++++++ .../qqq/backend/core/utils/TestUtils.java | 4 +- 4 files changed, 178 insertions(+), 14 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java index ce0a0920..2ca95347 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java @@ -23,8 +23,12 @@ package com.kingsrook.qqq.backend.core.actions.reporting; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager; @@ -32,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; @@ -41,10 +46,13 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -95,15 +103,25 @@ public class ExportAction /////////////////////////////////// if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames())) { - QTableMetaData table = exportInput.getTable(); - List badFieldNames = new ArrayList<>(); + QTableMetaData table = exportInput.getTable(); + Map joinTableMap = getJoinTableMap(table); + + List badFieldNames = new ArrayList<>(); for(String fieldName : exportInput.getFieldNames()) { try { - table.getField(fieldName); + if(fieldName.contains(".")) + { + String[] parts = fieldName.split("\\.", 2); + joinTableMap.get(parts[0]).getField(parts[1]); + } + else + { + table.getField(fieldName); + } } - catch(IllegalArgumentException iae) + catch(Exception e) { badFieldNames.add(fieldName); } @@ -128,6 +146,21 @@ public class ExportAction + /******************************************************************************* + ** + *******************************************************************************/ + private static Map getJoinTableMap(QTableMetaData table) + { + Map joinTableMap = new HashMap<>(); + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + joinTableMap.put(exposedJoin.getJoinTable(), QContext.getQInstance().getTable(exposedJoin.getJoinTable())); + } + return joinTableMap; + } + + + /******************************************************************************* ** Run the report. *******************************************************************************/ @@ -151,7 +184,33 @@ public class ExportAction QueryInput queryInput = new QueryInput(); queryInput.setTableName(exportInput.getTableName()); queryInput.setFilter(exportInput.getQueryFilter()); - queryInput.setLimit(exportInput.getLimit()); + + List queryJoins = new ArrayList<>(); + Set addedJoinNames = new HashSet<>(); + if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames())) + { + for(String fieldName : exportInput.getFieldNames()) + { + if(fieldName.contains(".")) + { + String[] parts = fieldName.split("\\.", 2); + String joinTableName = parts[0]; + if(!addedJoinNames.contains(joinTableName)) + { + queryJoins.add(new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true)); + addedJoinNames.add(joinTableName); + } + } + } + } + + queryInput.setQueryJoins(queryJoins); + + if(queryInput.getFilter() == null) + { + queryInput.setFilter(new QQueryFilter()); + } + queryInput.getFilter().setLimit(exportInput.getLimit()); queryInput.setShouldTranslatePossibleValues(true); ///////////////////////////////////////////////////////////////// @@ -298,11 +357,29 @@ public class ExportAction *******************************************************************************/ private List getFields(ExportInput exportInput) { + QTableMetaData table = exportInput.getTable(); + Map joinTableMap = getJoinTableMap(table); + List fieldList; - QTableMetaData table = exportInput.getTable(); if(exportInput.getFieldNames() != null) { - fieldList = exportInput.getFieldNames().stream().map(table::getField).toList(); + fieldList = new ArrayList<>(); + for(String fieldName : exportInput.getFieldNames()) + { + if(fieldName.contains(".")) + { + String[] parts = fieldName.split("\\.", 2); + QTableMetaData joinTable = joinTableMap.get(parts[0]); + QFieldMetaData field = joinTable.getField(parts[1]).clone(); + field.setName(fieldName); + field.setLabel(joinTable.getLabel() + ": " + field.getLabel()); + fieldList.add(field); + } + else + { + fieldList.add(table.getField(fieldName)); + } + } } else { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java index 6cc00b8f..b25cac78 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java @@ -38,13 +38,16 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -181,7 +184,7 @@ public class MemoryRecordStore } BackendQueryFilterUtils.sortRecordList(input.getFilter(), records); - records = BackendQueryFilterUtils.applySkipAndLimit(input, records); + records = BackendQueryFilterUtils.applySkipAndLimit(input.getFilter(), records); return (records); } @@ -191,8 +194,11 @@ public class MemoryRecordStore /******************************************************************************* ** *******************************************************************************/ - private Collection buildJoinCrossProduct(QueryInput input) + private Collection buildJoinCrossProduct(QueryInput input) throws QException { + QInstance qInstance = QContext.getQInstance(); + JoinsContext joinsContext = new JoinsContext(qInstance, input.getTableName(), input.getQueryJoins(), input.getFilter()); + List crossProduct = new ArrayList<>(); QTableMetaData leftTable = input.getTable(); for(QRecord record : getTableData(leftTable).values()) @@ -204,16 +210,26 @@ public class MemoryRecordStore for(QueryJoin queryJoin : input.getQueryJoins()) { - QTableMetaData nextTable = QContext.getQInstance().getTable(queryJoin.getJoinTable()); + QTableMetaData nextTable = qInstance.getTable(queryJoin.getJoinTable()); Collection nextTableRecords = getTableData(nextTable).values(); + QJoinMetaData joinMetaData = Objects.requireNonNullElseGet(queryJoin.getJoinMetaData(), () -> + { + QJoinMetaData found = joinsContext.findJoinMetaData(qInstance, input.getTableName(), queryJoin.getJoinTable()); + if(found == null) + { + throw (new RuntimeException("Could not find a join between tables [" + input.getTableName() + "][" + queryJoin.getJoinTable() + "]")); + } + return (found); + }); + List nextLevelProduct = new ArrayList<>(); for(QRecord productRecord : crossProduct) { boolean matchFound = false; for(QRecord nextTableRecord : nextTableRecords) { - if(joinMatches(productRecord, nextTableRecord, queryJoin)) + if(joinMatches(productRecord, nextTableRecord, queryJoin, joinMetaData)) { QRecord joinRecord = new QRecord(productRecord); addRecordToProduct(joinRecord, nextTableRecord, queryJoin.getJoinTableOrItsAlias()); @@ -239,9 +255,9 @@ public class MemoryRecordStore /******************************************************************************* ** *******************************************************************************/ - private boolean joinMatches(QRecord productRecord, QRecord nextTableRecord, QueryJoin queryJoin) + private boolean joinMatches(QRecord productRecord, QRecord nextTableRecord, QueryJoin queryJoin, QJoinMetaData joinMetaData) { - for(JoinOn joinOn : queryJoin.getJoinMetaData().getJoinOns()) + for(JoinOn joinOn : joinMetaData.getJoinOns()) { Serializable leftValue = productRecord.getValues().containsKey(queryJoin.getBaseTableOrAlias() + "." + joinOn.getLeftField()) ? productRecord.getValue(queryJoin.getBaseTableOrAlias() + "." + joinOn.getLeftField()) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java index a4fccc5c..9ff5f7de 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java @@ -22,10 +22,12 @@ package com.kingsrook.qqq.backend.core.actions.reporting; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.BaseTest; @@ -36,16 +38,21 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; import org.apache.commons.io.FileUtils; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -117,6 +124,68 @@ class ExportActionTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testJoins() throws QException, IOException + { + QInstance qInstance = QContext.getQInstance(); + QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true); + + TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_ORDER), List.of( + new QRecord().withValue("id", 1).withValue("orderNo", "ORD1").withValue("storeId", 1), + new QRecord().withValue("id", 2).withValue("orderNo", "ORD2").withValue("storeId", 1) + )); + + TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_LINE_ITEM), List.of( + new QRecord().withValue("id", 1).withValue("orderId", 1).withValue("sku", "A").withValue("quantity", 10), + new QRecord().withValue("id", 2).withValue("orderId", 1).withValue("sku", "B").withValue("quantity", 15), + new QRecord().withValue("id", 3).withValue("orderId", 2).withValue("sku", "A").withValue("quantity", 20) + )); + + ExportInput exportInput = new ExportInput(); + exportInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + exportInput.setReportFormat(ReportFormat.CSV); + ByteArrayOutputStream reportOutputStream = new ByteArrayOutputStream(); + exportInput.setReportOutputStream(reportOutputStream); + exportInput.setQueryFilter(new QQueryFilter()); + exportInput.setFieldNames(List.of("id", "orderNo", "storeId", "orderLine.id", "orderLine.sku", "orderLine.quantity")); + // exportInput.setFieldNames(List.of("id", "orderNo", "storeId")); + new ExportAction().execute(exportInput); + + String csv = reportOutputStream.toString(StandardCharsets.UTF_8); + CSVParser parse = CSVParser.parse(csv, CSVFormat.DEFAULT.withFirstRecordAsHeader()); + Iterator csvRecordIterator = parse.iterator(); + assertFalse(parse.getHeaderMap().isEmpty()); + assertTrue(parse.getHeaderMap().containsKey("Id")); + assertTrue(parse.getHeaderMap().containsKey("Order Line: Id")); + assertTrue(parse.getHeaderMap().containsKey("Order Line: SKU")); + + CSVRecord csvRecord = csvRecordIterator.next(); + assertEquals("1", csvRecord.get("Id")); + assertEquals("1", csvRecord.get("Order Line: Id")); + assertEquals("A", csvRecord.get("Order Line: SKU")); + assertEquals("10", csvRecord.get("Order Line: Quantity")); + + csvRecord = csvRecordIterator.next(); + assertEquals("1", csvRecord.get("Id")); + assertEquals("2", csvRecord.get("Order Line: Id")); + assertEquals("B", csvRecord.get("Order Line: SKU")); + assertEquals("15", csvRecord.get("Order Line: Quantity")); + + csvRecord = csvRecordIterator.next(); + assertEquals("2", csvRecord.get("Id")); + assertEquals("3", csvRecord.get("Order Line: Id")); + assertEquals("A", csvRecord.get("Order Line: SKU")); + assertEquals("20", csvRecord.get("Order Line: Quantity")); + + assertFalse(csvRecordIterator.hasNext()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 70b74a32..1fad436f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -92,6 +92,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTracking; @@ -547,6 +548,7 @@ public class TestUtils .withFieldName("storeId")) .withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_LINE_ITEM).withJoinName("orderLineItem")) .withAssociation(new Association().withName("extrinsics").withAssociatedTableName(TABLE_NAME_ORDER_EXTRINSIC).withJoinName("orderOrderExtrinsic")) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_LINE_ITEM)) .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) @@ -581,7 +583,7 @@ public class TestUtils .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("orderId", QFieldType.INTEGER)) .withField(new QFieldMetaData("lineNumber", QFieldType.STRING)) - .withField(new QFieldMetaData("sku", QFieldType.STRING)) + .withField(new QFieldMetaData("sku", QFieldType.STRING).withLabel("SKU")) .withField(new QFieldMetaData("quantity", QFieldType.INTEGER)); } From e0e45197082edc28099203b5c75a18f5e7a24aeb Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Apr 2023 10:21:22 -0500 Subject: [PATCH 13/15] Downgrade some logs --- .../core/actions/audits/DMLAuditAction.java | 2 +- .../values/QPossibleValueTranslator.java | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java index c103cc73..8c2f747e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java @@ -239,7 +239,7 @@ public class DMLAuditAction extends AbstractQActionFunction id; - case "label" -> label; - default -> throw new IllegalArgumentException("Unexpected value field: " + valueField); - }; + { + case "id" -> id; + case "label" -> label; + default -> throw new IllegalArgumentException("Unexpected value field: " + valueField); + }; values.add(Objects.requireNonNullElse(value, "")); } } @@ -427,7 +427,7 @@ public class QPossibleValueTranslator int size = entry.getValue().size(); if(size > 50_000) { - LOG.debug("Found a big PVS cache - clearing it.", logPair("name", entry.getKey()), logPair("size", size)); + LOG.info("Found a big PVS cache - clearing it.", logPair("name", entry.getKey()), logPair("size", size)); } } @@ -483,7 +483,7 @@ public class QPossibleValueTranslator { if(limitedToFieldNames != null && !limitedToFieldNames.contains(fieldNamePrefix + field.getName())) { - LOG.debug("Skipping cache priming for translation of possible value field [" + fieldNamePrefix + field.getName() + "] - it's not in the limitedToFieldNames set."); + LOG.trace("Skipping cache priming for translation of possible value field [" + fieldNamePrefix + field.getName() + "] - it's not in the limitedToFieldNames set."); continue; } @@ -556,7 +556,7 @@ public class QPossibleValueTranslator queryInput.setFieldsToTranslatePossibleValues(possibleValueFieldsToTranslate); } - LOG.debug("Priming PVS cache for [" + page.size() + "] ids from [" + tableName + "] table."); + LOG.trace("Priming PVS cache for [" + page.size() + "] ids from [" + tableName + "] table."); QueryOutput queryOutput = new QueryAction().execute(queryInput); /////////////////////////////////////////////////////////////////////////////////// From 5070502bded3280c2a81b883e103141b6ca70dcb Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 1 May 2023 14:17:08 -0500 Subject: [PATCH 14/15] Search for join meta data through exposed joins. --- .../actions/tables/query/JoinsContext.java | 200 ++++++++++++++++-- .../rdbms/actions/AbstractRDBMSAction.java | 58 ++++- .../qqq/backend/module/rdbms/TestUtils.java | 4 + .../rdbms/actions/RDBMSQueryActionTest.java | 158 ++++++++++++++ .../test/resources/prime-test-database.sql | 19 +- 5 files changed, 413 insertions(+), 26 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java index 5a66e1e3..2a52e634 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java @@ -32,12 +32,16 @@ import java.util.Objects; import java.util.Set; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.collections.MutableList; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -46,6 +50,8 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; *******************************************************************************/ public class JoinsContext { + private static final QLogger LOG = QLogger.getLogger(JoinsContext.class); + private final QInstance instance; private final String mainTableName; private final List queryJoins; @@ -65,7 +71,7 @@ public class JoinsContext { this.instance = instance; this.mainTableName = tableName; - this.queryJoins = CollectionUtils.nonNullList(queryJoins); + this.queryJoins = new MutableList<>(queryJoins); for(QueryJoin queryJoin : this.queryJoins) { @@ -123,6 +129,8 @@ public class JoinsContext ensureFilterIsRepresented(filter); + addJoinsFromExposedJoinPaths(); + /* todo!! for(QueryJoin queryJoin : queryJoins) { @@ -132,7 +140,147 @@ public class JoinsContext // addCriteriaForRecordSecurityLock(instance, session, joinTable, securityCriteria, recordSecurityLock, joinsContext, queryJoin.getJoinTableOrItsAlias()); } } - */ + */ + } + + + + /******************************************************************************* + ** If there are any joins in the context that don't have a join meta data, see + ** if we can find the JoinMetaData to use for them by looking at the main table's + ** exposed joins, and using their join paths. + *******************************************************************************/ + private void addJoinsFromExposedJoinPaths() throws QException + { + //////////////////////////////////////////////////////////////////////////////// + // do a double-loop, to avoid concurrent modification on the queryJoins list. // + // that is to say, we'll loop over that list, but possibly add things to it, // + // in which case we'll set this flag, and break the inner loop, to go again. // + //////////////////////////////////////////////////////////////////////////////// + boolean addedJoin; + do + { + addedJoin = false; + for(QueryJoin queryJoin : queryJoins) + { + ///////////////////////////////////////////////////////////////////// + // if the join has joinMetaData, then we don't need to process it. // + ///////////////////////////////////////////////////////////////////// + if(queryJoin.getJoinMetaData() == null) + { + ////////////////////////////////////////////////////////////////////// + // try to find a direct join between the main table and this table. // + // if one is found, then put it (the meta data) on the query join. // + ////////////////////////////////////////////////////////////////////// + String baseTableName = Objects.requireNonNullElse(resolveTableNameOrAliasToTableName(queryJoin.getBaseTableOrAlias()), mainTableName); + QJoinMetaData found = findJoinMetaData(instance, baseTableName, queryJoin.getJoinTable()); + if(found != null) + { + queryJoin.setJoinMetaData(found); + } + else + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else, the join must be indirect - so look for an exposedJoin that will have a joinPath that will connect us // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + LOG.debug("Looking for an exposed join...", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable())); + + QTableMetaData mainTable = instance.getTable(mainTableName); + boolean addedAnyQueryJoins = false; + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(mainTable.getExposedJoins())) + { + if(queryJoin.getJoinTable().equals(exposedJoin.getJoinTable())) + { + LOG.debug("Found an exposed join", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()), logPair("joinPath", exposedJoin.getJoinPath())); + + ///////////////////////////////////////////////////////////////////////////////////// + // loop backward through the join path (from the joinTable back to the main table) // + // adding joins to the table (if they aren't already in the query) // + ///////////////////////////////////////////////////////////////////////////////////// + String tmpTable = queryJoin.getJoinTable(); + for(int i = exposedJoin.getJoinPath().size() - 1; i >= 0; i--) + { + String joinName = exposedJoin.getJoinPath().get(i); + QJoinMetaData joinToAdd = instance.getJoin(joinName); + + ///////////////////////////////////////////////////////////////////////////// + // get the name from the opposite side of the join (flipping it if needed) // + ///////////////////////////////////////////////////////////////////////////// + String nextTable; + if(joinToAdd.getRightTable().equals(tmpTable)) + { + nextTable = joinToAdd.getLeftTable(); + } + else + { + nextTable = joinToAdd.getRightTable(); + joinToAdd = joinToAdd.flip(); + } + + if(doesJoinNeedAddedToQuery(joinName)) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if this is the last element in the joinPath, then we want to set this joinMetaData on the outer queryJoin // + // - else, we need to add a new queryJoin to this context // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(i == exposedJoin.getJoinPath().size() - 1) + { + if(queryJoin.getBaseTableOrAlias() == null) + { + queryJoin.setBaseTableOrAlias(nextTable); + } + queryJoin.setJoinMetaData(joinToAdd); + } + else + { + QueryJoin queryJoinToAdd = makeQueryJoinFromJoinAndTableNames(nextTable, tmpTable, joinToAdd); + queryJoinToAdd.setType(queryJoin.getType()); + addedAnyQueryJoins = true; + this.queryJoins.add(queryJoinToAdd); // todo something else with aliases? probably. + processQueryJoin(queryJoinToAdd); + } + } + + tmpTable = nextTable; + } + } + } + + /////////////////////////////////////////////////////////////////////////////////////////////////// + // break the inner loop (it would fail due to a concurrent modification), but continue the outer // + /////////////////////////////////////////////////////////////////////////////////////////////////// + if(addedAnyQueryJoins) + { + addedJoin = true; + break; + } + } + } + } + } + while(addedJoin); + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean doesJoinNeedAddedToQuery(String joinName) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // look at all queryJoins already in context - if any have this join's name, then we don't need this join... // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QueryJoin queryJoin : queryJoins) + { + if(queryJoin.getJoinMetaData() != null && queryJoin.getJoinMetaData().getName().equals(joinName)) + { + return (false); + } + } + + return (true); } @@ -256,34 +404,54 @@ public class JoinsContext { if(!aliasToTableNameMap.containsKey(filterTable) && !Objects.equals(mainTableName, filterTable)) { + boolean found = false; for(QJoinMetaData join : CollectionUtils.nonNullMap(QContext.getQInstance().getJoins()).values()) { - QueryJoin queryJoin = null; - if(join.getLeftTable().equals(mainTableName) && join.getRightTable().equals(filterTable)) - { - queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER); - } - else - { - join = join.flip(); - if(join.getLeftTable().equals(mainTableName) && join.getRightTable().equals(filterTable)) - { - queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER); - } - } - + QueryJoin queryJoin = makeQueryJoinFromJoinAndTableNames(mainTableName, filterTable, join); if(queryJoin != null) { this.queryJoins.add(queryJoin); // todo something else with aliases? probably. processQueryJoin(queryJoin); + found = true; + break; } } + + if(!found) + { + QueryJoin queryJoin = new QueryJoin().withJoinTable(filterTable).withType(QueryJoin.Type.INNER); + this.queryJoins.add(queryJoin); // todo something else with aliases? probably. + processQueryJoin(queryJoin); + } } } } + /******************************************************************************* + ** + *******************************************************************************/ + private QueryJoin makeQueryJoinFromJoinAndTableNames(String tableA, String tableB, QJoinMetaData join) + { + QueryJoin queryJoin = null; + if(join.getLeftTable().equals(tableA) && join.getRightTable().equals(tableB)) + { + queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER); + } + else + { + join = join.flip(); + if(join.getLeftTable().equals(tableA) && join.getRightTable().equals(tableB)) + { + queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER); + } + } + return queryJoin; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index fc011387..ec7e7488 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -30,9 +30,12 @@ import java.time.Instant; import java.time.LocalDate; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; @@ -203,7 +206,8 @@ public abstract class AbstractRDBMSAction implements QActionInterface { StringBuilder rs = new StringBuilder(escapeIdentifier(getTableName(instance.getTable(tableName))) + " AS " + escapeIdentifier(tableName)); - for(QueryJoin queryJoin : joinsContext.getQueryJoins()) + List queryJoins = sortQueryJoinsForFromClause(tableName, joinsContext.getQueryJoins()); + for(QueryJoin queryJoin : queryJoins) { QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable()); String tableNameOrAlias = queryJoin.getJoinTableOrItsAlias(); @@ -264,6 +268,48 @@ public abstract class AbstractRDBMSAction implements QActionInterface + /******************************************************************************* + ** We've seen some SQL dialects (mysql, but not h2...) be unhappy if we have + ** the from/join-ons out of order. This method resorts the joins, to start with + ** main table, then any tables attached to it, then fanning out from there. + *******************************************************************************/ + private List sortQueryJoinsForFromClause(String mainTableName, List queryJoins) + { + List inputListCopy = new ArrayList<>(queryJoins); + + List rs = new ArrayList<>(); + Set seenTables = new HashSet<>(); + seenTables.add(mainTableName); + + boolean keepGoing = true; + while(!inputListCopy.isEmpty() && keepGoing) + { + keepGoing = false; + Iterator iterator = inputListCopy.iterator(); + while(iterator.hasNext()) + { + QueryJoin next = iterator.next(); + if((StringUtils.hasContent(next.getBaseTableOrAlias()) && seenTables.contains(next.getBaseTableOrAlias())) || seenTables.contains(next.getJoinTable())) + { + rs.add(next); + if(StringUtils.hasContent(next.getBaseTableOrAlias())) + { + seenTables.add(next.getBaseTableOrAlias()); + } + seenTables.add(next.getJoinTable()); + iterator.remove(); + keepGoing = true; + } + } + } + + rs.addAll(inputListCopy); + + return (rs); + } + + + /******************************************************************************* ** method that sub-classes should call to make a full WHERE clause, including ** security clauses. @@ -902,6 +948,16 @@ public abstract class AbstractRDBMSAction implements QActionInterface + /******************************************************************************* + ** Make it easy (e.g., for tests) to turn on logging of SQL + *******************************************************************************/ + public static void setLogSQLOutput(String loggerOrSystemOut) + { + System.setProperty("qqq.rdbms.logSQL.output", loggerOrSystemOut); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java index 0dad47e0..a2479159 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java @@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; @@ -239,6 +240,7 @@ public class TestUtils qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER, "order") .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId")) .withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_ORDER_LINE).withJoinName("orderJoinOrderLine")) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ITEM).withJoinPath(List.of("orderJoinOrderLine", "orderLineJoinItem"))) .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE)) .withField(new QFieldMetaData("billToPersonId", QFieldType.INTEGER).withBackendName("bill_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) .withField(new QFieldMetaData("shipToPersonId", QFieldType.INTEGER).withBackendName("ship_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) @@ -246,7 +248,9 @@ public class TestUtils qInstance.addTable(defineBaseTable(TABLE_NAME_ITEM, "item") .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId")) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER).withJoinPath(List.of("orderLineJoinItem", "orderJoinOrderLine"))) .withField(new QFieldMetaData("sku", QFieldType.STRING)) + .withField(new QFieldMetaData("description", QFieldType.STRING)) .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE)) ); diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java index 4d846e19..637601fa 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java @@ -51,6 +51,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -72,6 +73,20 @@ public class RDBMSQueryActionTest extends RDBMSActionTest public void beforeEach() throws Exception { super.primeTestDatabase(); + + // AbstractRDBMSAction.setLogSQL(true); + // AbstractRDBMSAction.setLogSQLOutput("system.out"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + AbstractRDBMSAction.setLogSQL(false); } @@ -1103,6 +1118,149 @@ public class RDBMSQueryActionTest extends RDBMSActionTest + /******************************************************************************* + ** Given tables: + ** order - orderLine - item + ** with exposedJoin on order to item + ** do a query on order, also selecting item. + *******************************************************************************/ + @Test + void testTwoTableAwayExposedJoin() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + queryInput.withQueryJoins(List.of( + new QueryJoin(TestUtils.TABLE_NAME_ITEM).withType(QueryJoin.Type.INNER).withSelect(true) + )); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + assertThat(records).hasSize(11); // one per line item + assertThat(records).allMatch(r -> r.getValue("id") != null); + assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ITEM + ".description") != null); + } + + + + /******************************************************************************* + ** Given tables: + ** order - orderLine - item + ** with exposedJoin on item to order + ** do a query on item, also selecting order. + ** This is a reverse of the above, to make sure join flipping, etc, is good. + *******************************************************************************/ + @Test + void testTwoTableAwayExposedJoinReversed() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ITEM); + + queryInput.withQueryJoins(List.of( + new QueryJoin(TestUtils.TABLE_NAME_ORDER).withType(QueryJoin.Type.INNER).withSelect(true) + )); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + assertThat(records).hasSize(11); // one per line item + assertThat(records).allMatch(r -> r.getValue("description") != null); + assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ORDER + ".id") != null); + } + + + + /******************************************************************************* + ** Given tables: + ** order - orderLine - item + ** with exposedJoin on order to item + ** do a query on order, also selecting item, and also selecting orderLine... + *******************************************************************************/ + @Test + void testTwoTableAwayExposedJoinAlsoSelectingInBetweenTable() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + queryInput.withQueryJoins(List.of( + new QueryJoin(TestUtils.TABLE_NAME_ORDER_LINE).withType(QueryJoin.Type.INNER).withSelect(true), + new QueryJoin(TestUtils.TABLE_NAME_ITEM).withType(QueryJoin.Type.INNER).withSelect(true) + )); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + assertThat(records).hasSize(11); // one per line item + assertThat(records).allMatch(r -> r.getValue("id") != null); + assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ORDER_LINE + ".quantity") != null); + assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ITEM + ".description") != null); + } + + + + /******************************************************************************* + ** Given tables: + ** order - orderLine - item + ** with exposedJoin on order to item + ** do a query on order, filtered by item + *******************************************************************************/ + @Test + void testTwoTableAwayExposedJoinWhereClauseOnly() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria(TestUtils.TABLE_NAME_ITEM + ".description", QCriteriaOperator.STARTS_WITH, "Q-Mart"))); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + assertThat(records).hasSize(4); + assertThat(records).allMatch(r -> r.getValue("id") != null); + } + + + + /******************************************************************************* + ** Given tables: + ** order - orderLine - item + ** with exposedJoin on order to item + ** do a query on order, filtered by item + *******************************************************************************/ + @Test + void testTwoTableAwayExposedJoinWhereClauseBothJoinTables() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria(TestUtils.TABLE_NAME_ITEM + ".description", QCriteriaOperator.STARTS_WITH, "Q-Mart")) + .withCriteria(new QFilterCriteria(TestUtils.TABLE_NAME_ORDER_LINE + ".quantity", QCriteriaOperator.IS_NOT_BLANK)) + ); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + assertThat(records).hasSize(4); + assertThat(records).allMatch(r -> r.getValue("id") != null); + } + + + /******************************************************************************* ** queries on the store table, where the primary key (id) is the security field *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql b/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql index eb2abfe2..92c69194 100644 --- a/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql +++ b/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql @@ -102,19 +102,20 @@ CREATE TABLE item ( id INT AUTO_INCREMENT PRIMARY KEY, sku VARCHAR(80) NOT NULL, + description VARCHAR(80), store_id INT NOT NULL REFERENCES store ); -- three items for each store -INSERT INTO item (id, sku, store_id) VALUES (1, 'QM-1', 1); -INSERT INTO item (id, sku, store_id) VALUES (2, 'QM-2', 1); -INSERT INTO item (id, sku, store_id) VALUES (3, 'QM-3', 1); -INSERT INTO item (id, sku, store_id) VALUES (4, 'QRU-1', 2); -INSERT INTO item (id, sku, store_id) VALUES (5, 'QRU-2', 2); -INSERT INTO item (id, sku, store_id) VALUES (6, 'QRU-3', 2); -INSERT INTO item (id, sku, store_id) VALUES (7, 'QD-1', 3); -INSERT INTO item (id, sku, store_id) VALUES (8, 'QD-2', 3); -INSERT INTO item (id, sku, store_id) VALUES (9, 'QD-3', 3); +INSERT INTO item (id, sku, description, store_id) VALUES (1, 'QM-1', 'Q-Mart Item 1', 1); +INSERT INTO item (id, sku, description, store_id) VALUES (2, 'QM-2', 'Q-Mart Item 2', 1); +INSERT INTO item (id, sku, description, store_id) VALUES (3, 'QM-3', 'Q-Mart Item 3', 1); +INSERT INTO item (id, sku, description, store_id) VALUES (4, 'QRU-1', 'QQQ R Us Item 4', 2); +INSERT INTO item (id, sku, description, store_id) VALUES (5, 'QRU-2', 'QQQ R Us Item 5', 2); +INSERT INTO item (id, sku, description, store_id) VALUES (6, 'QRU-3', 'QQQ R Us Item 6', 2); +INSERT INTO item (id, sku, description, store_id) VALUES (7, 'QD-1', 'QDepot Item 7', 3); +INSERT INTO item (id, sku, description, store_id) VALUES (8, 'QD-2', 'QDepot Item 8', 3); +INSERT INTO item (id, sku, description, store_id) VALUES (9, 'QD-3', 'QDepot Item 9', 3); CREATE TABLE `order` ( From 1a8ab34fe309d069c04c3bbe084cadca06fc7b3a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 1 May 2023 14:17:25 -0500 Subject: [PATCH 15/15] make a new collection if given a null one as input. --- .../core/utils/collections/MutableList.java | 29 +++++++++++++------ .../core/utils/collections/MutableMap.java | 27 ++++++++++++----- .../utils/collections/MutableListTest.java | 20 +++++++++++++ .../utils/collections/MutableMapTest.java | 20 +++++++++++++ 4 files changed, 79 insertions(+), 17 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableList.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableList.java index b43fd241..78084bbc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableList.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableList.java @@ -27,6 +27,7 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.ListIterator; +import java.util.Objects; import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.utils.lambdas.VoidVoidMethod; @@ -38,8 +39,8 @@ import com.kingsrook.qqq.backend.core.utils.lambdas.VoidVoidMethod; *******************************************************************************/ public class MutableList implements List { - private List sourceList; - private Class> mutableTypeIfNeeded; + private List sourceList; + private Supplier> supplierIfNeeded; @@ -49,7 +50,7 @@ public class MutableList implements List *******************************************************************************/ public MutableList(List sourceList) { - this(sourceList, (Class) ArrayList.class); + this(sourceList, ArrayList::new); } @@ -58,10 +59,20 @@ public class MutableList implements List ** Constructor ** *******************************************************************************/ - public MutableList(List sourceList, Class> mutableTypeIfNeeded) + public MutableList(List sourceList, Supplier> supplierIfNeeded) { - this.sourceList = sourceList; - this.mutableTypeIfNeeded = mutableTypeIfNeeded; + this.sourceList = Objects.requireNonNullElseGet(sourceList, supplierIfNeeded); + this.supplierIfNeeded = supplierIfNeeded; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + List getUnderlyingList() + { + return (sourceList); } @@ -73,13 +84,13 @@ public class MutableList implements List { try { - List replacementList = mutableTypeIfNeeded.getConstructor().newInstance(); + List replacementList = supplierIfNeeded.get(); replacementList.addAll(sourceList); sourceList = replacementList; } catch(Exception e) { - throw (new IllegalStateException("The mutable type provided for this MutableList [" + mutableTypeIfNeeded.getName() + "] could not be instantiated.")); + throw (new IllegalStateException("Error getting from the supplier provided for this MutableList.", e)); } } @@ -88,7 +99,7 @@ public class MutableList implements List /******************************************************************************* ** *******************************************************************************/ - private T doMutableOperationForValue(Supplier supplier) + private V doMutableOperationForValue(Supplier supplier) { try { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMap.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMap.java index 79102a07..87789756 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMap.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMap.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.utils.collections; import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.utils.lambdas.VoidVoidMethod; @@ -37,8 +38,8 @@ import com.kingsrook.qqq.backend.core.utils.lambdas.VoidVoidMethod; *******************************************************************************/ public class MutableMap implements Map { - private Map sourceMap; - private Class> mutableTypeIfNeeded; + private Map sourceMap; + private Supplier> supplierIfNeeded; @@ -48,7 +49,7 @@ public class MutableMap implements Map *******************************************************************************/ public MutableMap(Map sourceMap) { - this(sourceMap, (Class) HashMap.class); + this(sourceMap, HashMap::new); } @@ -57,10 +58,20 @@ public class MutableMap implements Map ** Constructor ** *******************************************************************************/ - public MutableMap(Map sourceMap, Class> mutableTypeIfNeeded) + public MutableMap(Map sourceMap, Supplier> supplierIfNeeded) { - this.sourceMap = sourceMap; - this.mutableTypeIfNeeded = mutableTypeIfNeeded; + this.sourceMap = Objects.requireNonNullElseGet(sourceMap, supplierIfNeeded); + this.supplierIfNeeded = supplierIfNeeded; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + Map getUnderlyingMap() + { + return (sourceMap); } @@ -72,13 +83,13 @@ public class MutableMap implements Map { try { - Map replacementMap = mutableTypeIfNeeded.getConstructor().newInstance(); + Map replacementMap = supplierIfNeeded.get(); replacementMap.putAll(sourceMap); sourceMap = replacementMap; } catch(Exception e) { - throw (new IllegalStateException("The mutable type provided for this MutableMap [" + mutableTypeIfNeeded.getName() + "] could not be instantiated.")); + throw (new IllegalStateException("Error getting from the supplier provided for this MutableMap.", e)); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableListTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableListTest.java index 677509fd..6136f814 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableListTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableListTest.java @@ -22,9 +22,11 @@ package com.kingsrook.qqq.backend.core.utils.collections; +import java.util.LinkedList; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -48,4 +50,22 @@ class MutableListTest extends BaseTest list.remove(0); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNullInput() + { + List list = new MutableList<>(null); + list.add(1); + assertEquals(1, list.size()); + + MutableList mutableList = new MutableList<>(null, LinkedList::new); + mutableList.add(1); + assertEquals(1, mutableList.size()); + assertEquals(LinkedList.class, mutableList.getUnderlyingList().getClass()); + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMapTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMapTest.java index cc2d160e..44e85544 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMapTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMapTest.java @@ -22,9 +22,11 @@ package com.kingsrook.qqq.backend.core.utils.collections; +import java.util.LinkedHashMap; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -48,4 +50,22 @@ class MutableMapTest extends BaseTest map.putAll(Map.of("c", 3, "d", 4)); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNullInput() + { + Map map = new MutableMap<>(null); + map.put(1, "one"); + assertEquals(1, map.size()); + + MutableMap mutableMap = new MutableMap<>(null, LinkedHashMap::new); + mutableMap.put(1, "uno"); + assertEquals(1, mutableMap.size()); + assertEquals(LinkedHashMap.class, mutableMap.getUnderlyingMap().getClass()); + } + } \ No newline at end of file