diff --git a/.circleci/config.yml b/.circleci/config.yml index da983b05..5981d2fa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,6 +26,7 @@ commands: sudo rm /etc/alternatives/java sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java - run: + ## used by jacoco uncovered class reporting in pom.xml name: Install html2text command: | sudo apt-get update @@ -73,11 +74,6 @@ commands: when: always - store_test_results: path: ~/test-results - - run: - name: Find Un-tested Classes - command: | - set +o pipefail && for i in */target/site/jacoco/*/index.html; do html2text -width 500 -nobs $i | sed '1,/^Total/d;' | grep -v Created | sed 's/ \+/ /g' | sed 's/ [[:digit:]]$//' | grep -v 0$ | cut -d' ' -f1; done - when: always - save_cache: paths: - ~/.m2 diff --git a/pom.xml b/pom.xml index 2bc0ecf0..4623b96d 100644 --- a/pom.xml +++ b/pom.xml @@ -206,6 +206,63 @@ true + + exec-maven-plugin + org.codehaus.mojo + 3.0.0 + + + test-coverage-summary + verify + + exec + + + sh + + -c + + /dev/null 2>&1 +if [ "$?" == "0" ]; then + echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers + xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values + paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}' + rm /tmp/$$.headers /tmp/$$.values +else + echo "xpath is not installed. Jacoco coverage summary will not be produced here..."; +fi + +which xpath > /dev/null 2>&1 +if [ "$?" == "0" ]; then + echo "Untested classes, per Jacoco:" + echo "-----------------------------" + for i in target/site/jacoco/*/index.html; do + html2text -width 500 -nobs $i | sed '1,/^Total/d;' | grep -v Created | sed 's/ \+/ /g' | sed 's/ [[:digit:]]$//' | grep -v 0$ | cut -d' ' -f1; + done; + echo +else + echo "html2text is not installed. Untested classes from Jacoco will not be printed here..."; +fi + + ]]> + + + + + + org.jacoco jacoco-maven-plugin @@ -249,56 +306,14 @@ post-unit-test - verify + + post-integration-test report - - exec-maven-plugin - org.codehaus.mojo - 3.0.0 - - - test-coverage-summary - verify - - exec - - - sh - - -c - - /dev/null 2>&1 -if [ "$?" == "0" ]; then - echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers - xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values - paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}' - rm /tmp/$$.headers /tmp/$$.values -else - echo "xpath is not installed. Jacoco coverage summary will not be produced here.."; -fi - ]]> - - - - - - diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java index c103cc73..8c2f747e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java @@ -239,7 +239,7 @@ public class DMLAuditAction extends AbstractQActionFunction. + */ + +package com.kingsrook.qqq.backend.core.actions.metadata; + + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** Object to represent the graph of joins in a QQQ Instance. e.g., all of the + ** connections among tables through joins. + *******************************************************************************/ +public class JoinGraph +{ + + private Set edges = new HashSet<>(); + + + + /******************************************************************************* + ** Graph edge (no graph nodes needed in here) + *******************************************************************************/ + private record Edge(String joinName, String leftTable, String rightTable) + { + } + + + + /******************************************************************************* + ** In this class, we are treating joins as non-directional graph edges - so - + ** use this class to "normalize" what may otherwise be duplicated joins in the + ** qInstance (e.g., A -> B and B -> A -- in the instance, those are valid, but + ** in our graph here, we want to consider those the same). + *******************************************************************************/ + private static class NormalizedJoin + { + private String tableA; + private String tableB; + private String joinFieldA; + private String joinFieldB; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public NormalizedJoin(QJoinMetaData joinMetaData) + { + boolean needFlip = false; + int tableCompare = joinMetaData.getLeftTable().compareTo(joinMetaData.getRightTable()); + if(tableCompare < 0) + { + needFlip = true; + } + else if(tableCompare == 0) + { + int fieldCompare = joinMetaData.getJoinOns().get(0).getLeftField().compareTo(joinMetaData.getJoinOns().get(0).getRightField()); + if(fieldCompare < 0) + { + needFlip = true; + } + } + + if(needFlip) + { + joinMetaData = joinMetaData.flip(); + } + + tableA = joinMetaData.getLeftTable(); + tableB = joinMetaData.getRightTable(); + joinFieldA = joinMetaData.getJoinOns().get(0).getLeftField(); + joinFieldB = joinMetaData.getJoinOns().get(0).getRightField(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean equals(Object o) + { + if(this == o) + { + return true; + } + if(o == null || getClass() != o.getClass()) + { + return false; + } + NormalizedJoin that = (NormalizedJoin) o; + return Objects.equals(tableA, that.tableA) && Objects.equals(tableB, that.tableB) && Objects.equals(joinFieldA, that.joinFieldA) && Objects.equals(joinFieldB, that.joinFieldB); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int hashCode() + { + return Objects.hash(tableA, tableB, joinFieldA, joinFieldB); + } + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public JoinGraph(QInstance qInstance) + { + Set usedJoins = new HashSet<>(); + for(QJoinMetaData join : CollectionUtils.nonNullMap(qInstance.getJoins()).values()) + { + NormalizedJoin normalizedJoin = new NormalizedJoin(join); + if(usedJoins.contains(normalizedJoin)) + { + continue; + } + + usedJoins.add(normalizedJoin); + edges.add(new Edge(join.getName(), join.getLeftTable(), join.getRightTable())); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public record JoinConnection(String joinTable, String viaJoinName) implements Comparable + { + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int compareTo(JoinConnection that) + { + Comparator comparator = Comparator.comparing((JoinConnection jc) -> jc.joinTable()) + .thenComparing((JoinConnection jc) -> jc.viaJoinName()); + return (comparator.compare(this, that)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public record JoinConnectionList(List list) implements Comparable + { + + /******************************************************************************* + ** + *******************************************************************************/ + public JoinConnectionList copy() + { + return new JoinConnectionList(new ArrayList<>(list)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public int compareTo(JoinConnectionList that) + { + if(this.equals(that)) + { + return (0); + } + + for(int i = 0; i < Math.min(this.list.size(), that.list.size()); i++) + { + int comp = this.list.get(i).compareTo(that.list.get(i)); + if(comp != 0) + { + return (comp); + } + } + + return (this.list.size() - that.list.size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean matchesJoinPath(List joinPath) + { + if(list.size() != joinPath.size()) + { + return (false); + } + + for(int i = 0; i < list.size(); i++) + { + if(!list.get(i).viaJoinName().equals(joinPath.get(i))) + { + return (false); + } + } + + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public String getJoinNamesAsString() + { + return (StringUtils.join(", ", list().stream().map(jc -> jc.viaJoinName()).toList())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public List getJoinNamesAsList() + { + return (list().stream().map(jc -> jc.viaJoinName()).toList()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Set getJoinConnections(String tableName) + { + Set rs = new TreeSet<>(); + doGetJoinConnections(rs, tableName, new ArrayList<>(), new JoinConnectionList(new ArrayList<>())); + return (rs); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void doGetJoinConnections(Set joinConnections, String tableName, List path, JoinConnectionList connectionList) + { + for(Edge edge : edges) + { + if(edge.leftTable.equals(tableName) || edge.rightTable.equals(tableName)) + { + if(path.contains(edge.joinName)) + { + continue; + } + + List newPath = new ArrayList<>(path); + newPath.add(edge.joinName); + if(!joinConnectionsContain(joinConnections, newPath)) + { + String otherTableName = null; + if(!edge.leftTable.equals(tableName)) + { + otherTableName = edge.leftTable; + } + else if(!edge.rightTable.equals(tableName)) + { + otherTableName = edge.rightTable; + } + + if(otherTableName != null) + { + + JoinConnectionList newConnectionList = connectionList.copy(); + JoinConnection joinConnection = new JoinConnection(otherTableName, edge.joinName); + newConnectionList.list.add(joinConnection); + joinConnections.add(newConnectionList); + doGetJoinConnections(joinConnections, otherTableName, new ArrayList<>(newPath), newConnectionList); + } + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean joinConnectionsContain(Set joinPaths, List newPath) + { + for(JoinConnectionList joinConnections : joinPaths) + { + List joinConnectionJoins = joinConnections.list.stream().map(jc -> jc.viaJoinName).toList(); + if(joinConnectionJoins.equals(newPath)) + { + return (true); + } + } + return (false); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java index 04dbc905..ef571e45 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java @@ -83,11 +83,14 @@ public class MetaDataAction } QBackendMetaData backendForTable = metaDataInput.getInstance().getBackendForTable(tableName); - tables.put(tableName, new QFrontendTableMetaData(metaDataInput, backendForTable, table, false)); + tables.put(tableName, new QFrontendTableMetaData(metaDataInput, backendForTable, table, false, false)); treeNodes.put(tableName, new AppTreeNode(table)); } metaDataOutput.setTables(tables); + // addJoinsToTables(tables); + // addJoinedTablesToTables(tables); + //////////////////////////////////////// // map processes to frontend metadata // //////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/TableMetaDataAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/TableMetaDataAction.java index 223eb475..46c4f243 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/TableMetaDataAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/TableMetaDataAction.java @@ -54,7 +54,7 @@ public class TableMetaDataAction throw (new QNotFoundException("Table [" + tableMetaDataInput.getTableName() + "] was not found.")); } QBackendMetaData backendForTable = tableMetaDataInput.getInstance().getBackendForTable(table.getName()); - tableMetaDataOutput.setTable(new QFrontendTableMetaData(tableMetaDataInput, backendForTable, table, true)); + tableMetaDataOutput.setTable(new QFrontendTableMetaData(tableMetaDataInput, backendForTable, table, true, true)); // todo post-customization - can do whatever w/ the result if you want diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java index ce0a0920..2ca95347 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java @@ -23,8 +23,12 @@ package com.kingsrook.qqq.backend.core.actions.reporting; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager; @@ -32,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; @@ -41,10 +46,13 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -95,15 +103,25 @@ public class ExportAction /////////////////////////////////// if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames())) { - QTableMetaData table = exportInput.getTable(); - List badFieldNames = new ArrayList<>(); + QTableMetaData table = exportInput.getTable(); + Map joinTableMap = getJoinTableMap(table); + + List badFieldNames = new ArrayList<>(); for(String fieldName : exportInput.getFieldNames()) { try { - table.getField(fieldName); + if(fieldName.contains(".")) + { + String[] parts = fieldName.split("\\.", 2); + joinTableMap.get(parts[0]).getField(parts[1]); + } + else + { + table.getField(fieldName); + } } - catch(IllegalArgumentException iae) + catch(Exception e) { badFieldNames.add(fieldName); } @@ -128,6 +146,21 @@ public class ExportAction + /******************************************************************************* + ** + *******************************************************************************/ + private static Map getJoinTableMap(QTableMetaData table) + { + Map joinTableMap = new HashMap<>(); + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + joinTableMap.put(exposedJoin.getJoinTable(), QContext.getQInstance().getTable(exposedJoin.getJoinTable())); + } + return joinTableMap; + } + + + /******************************************************************************* ** Run the report. *******************************************************************************/ @@ -151,7 +184,33 @@ public class ExportAction QueryInput queryInput = new QueryInput(); queryInput.setTableName(exportInput.getTableName()); queryInput.setFilter(exportInput.getQueryFilter()); - queryInput.setLimit(exportInput.getLimit()); + + List queryJoins = new ArrayList<>(); + Set addedJoinNames = new HashSet<>(); + if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames())) + { + for(String fieldName : exportInput.getFieldNames()) + { + if(fieldName.contains(".")) + { + String[] parts = fieldName.split("\\.", 2); + String joinTableName = parts[0]; + if(!addedJoinNames.contains(joinTableName)) + { + queryJoins.add(new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true)); + addedJoinNames.add(joinTableName); + } + } + } + } + + queryInput.setQueryJoins(queryJoins); + + if(queryInput.getFilter() == null) + { + queryInput.setFilter(new QQueryFilter()); + } + queryInput.getFilter().setLimit(exportInput.getLimit()); queryInput.setShouldTranslatePossibleValues(true); ///////////////////////////////////////////////////////////////// @@ -298,11 +357,29 @@ public class ExportAction *******************************************************************************/ private List getFields(ExportInput exportInput) { + QTableMetaData table = exportInput.getTable(); + Map joinTableMap = getJoinTableMap(table); + List fieldList; - QTableMetaData table = exportInput.getTable(); if(exportInput.getFieldNames() != null) { - fieldList = exportInput.getFieldNames().stream().map(table::getField).toList(); + fieldList = new ArrayList<>(); + for(String fieldName : exportInput.getFieldNames()) + { + if(fieldName.contains(".")) + { + String[] parts = fieldName.split("\\.", 2); + QTableMetaData joinTable = joinTableMap.get(parts[0]); + QFieldMetaData field = joinTable.getField(parts[1]).clone(); + field.setName(fieldName); + field.setLabel(joinTable.getLabel() + ": " + field.getLabel()); + fieldList.add(field); + } + else + { + fieldList.add(table.getField(fieldName)); + } + } } else { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java index 04b1d51f..5014caa2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java @@ -156,8 +156,7 @@ public class StoreAssociatedScriptAction queryInput.setFilter(new QQueryFilter() .withCriteria(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, List.of(script.getValue("id")))) .withOrderBy(new QFilterOrderBy("sequenceNo", false)) - ); - queryInput.setLimit(1); + .withLimit(1)); QueryOutput queryOutput = new QueryAction().execute(queryInput); if(!queryOutput.getRecords().isEmpty()) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java index 9d7ed1a7..d797e393 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java @@ -123,7 +123,7 @@ public class QPossibleValueTranslator return; } - LOG.debug("Translating possible values in [" + records.size() + "] records from the [" + table.getName() + "] table."); + LOG.trace("Translating possible values in [" + records.size() + "] records from the [" + table.getName() + "] table."); primePvsCache(table, records, queryJoins, limitedToFieldNames); for(QRecord record : records) @@ -378,11 +378,11 @@ public class QPossibleValueTranslator for(String valueField : valueFields) { Object value = switch(valueField) - { - case "id" -> id; - case "label" -> label; - default -> throw new IllegalArgumentException("Unexpected value field: " + valueField); - }; + { + case "id" -> id; + case "label" -> label; + default -> throw new IllegalArgumentException("Unexpected value field: " + valueField); + }; values.add(Objects.requireNonNullElse(value, "")); } } @@ -427,7 +427,7 @@ public class QPossibleValueTranslator int size = entry.getValue().size(); if(size > 50_000) { - LOG.debug("Found a big PVS cache - clearing it.", logPair("name", entry.getKey()), logPair("size", size)); + LOG.info("Found a big PVS cache - clearing it.", logPair("name", entry.getKey()), logPair("size", size)); } } @@ -483,7 +483,7 @@ public class QPossibleValueTranslator { if(limitedToFieldNames != null && !limitedToFieldNames.contains(fieldNamePrefix + field.getName())) { - LOG.debug("Skipping cache priming for translation of possible value field [" + fieldNamePrefix + field.getName() + "] - it's not in the limitedToFieldNames set."); + LOG.trace("Skipping cache priming for translation of possible value field [" + fieldNamePrefix + field.getName() + "] - it's not in the limitedToFieldNames set."); continue; } @@ -556,7 +556,7 @@ public class QPossibleValueTranslator queryInput.setFieldsToTranslatePossibleValues(possibleValueFieldsToTranslate); } - LOG.debug("Priming PVS cache for [" + page.size() + "] ids from [" + tableName + "] table."); + LOG.trace("Priming PVS cache for [" + page.size() + "] ids from [" + tableName + "] table."); QueryOutput queryOutput = new QueryAction().execute(queryInput); /////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java index c034e1f7..8a25136c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java @@ -247,7 +247,7 @@ public class SearchPossibleValueSourceAction queryFilter.setOrderBys(possibleValueSource.getOrderByFields()); // todo - skip & limit as params - queryInput.setLimit(250); + queryFilter.setLimit(250); /////////////////////////////////////////////////////////////////////////////////////////////////////// // if given a default filter, make it the 'top level' filter and the one we just created a subfilter // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index df6fca5a..38213836 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -32,7 +32,9 @@ import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph; import com.kingsrook.qqq.backend.core.actions.permissions.BulkTableActionProcessPermissionChecker; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -59,6 +61,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QMiddlewareTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -87,6 +90,8 @@ public class QInstanceEnricher private final QInstance qInstance; + private JoinGraph joinGraph; + ////////////////////////////////////////////////////////// // todo - come up w/ a way for app devs to set configs! // ////////////////////////////////////////////////////////// @@ -144,6 +149,81 @@ public class QInstanceEnricher { qInstance.getWidgets().values().forEach(this::enrichWidget); } + + enrichJoins(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void enrichJoins() + { + try + { + joinGraph = new JoinGraph(qInstance); + + for(QTableMetaData table : CollectionUtils.nonNullMap(qInstance.getTables()).values()) + { + Set joinConnections = joinGraph.getJoinConnections(table.getName()); + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // proceed with caution - remember, validator will fail the instance if things are missing/invalid // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + if(exposedJoin.getJoinTable() != null) + { + QTableMetaData joinTable = qInstance.getTable(exposedJoin.getJoinTable()); + if(joinTable != null) + { + ////////////////////////////////////////////////////////////////////////////////// + // default the exposed join's label to the join table's label, if it wasn't set // + ////////////////////////////////////////////////////////////////////////////////// + if(!StringUtils.hasContent(exposedJoin.getLabel())) + { + exposedJoin.setLabel(joinTable.getLabel()); + } + + /////////////////////////////////////////////////////////////////////////////// + // default the exposed join's join-path from the joinGraph, if it wasn't set // + /////////////////////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeIsEmpty(exposedJoin.getJoinPath())) + { + List eligibleJoinConnections = new ArrayList<>(); + for(JoinGraph.JoinConnectionList joinConnection : joinConnections) + { + if(joinTable.getName().equals(joinConnection.list().get(joinConnection.list().size() - 1).joinTable())) + { + eligibleJoinConnections.add(joinConnection); + } + } + + if(eligibleJoinConnections.isEmpty()) + { + throw (new QException("Could not infer a joinPath for table [" + table.getName() + "], exposedJoin to [" + exposedJoin.getJoinTable() + "]: No join connections between these tables exist in this instance.")); + } + else if(eligibleJoinConnections.size() > 1) + { + throw (new QException("Could not infer a joinPath for table [" + table.getName() + "], exposedJoin to [" + exposedJoin.getJoinTable() + "]: " + + eligibleJoinConnections.size() + " join connections exist between these tables. You need to specify one:\n" + + StringUtils.join("\n", eligibleJoinConnections.stream().map(jcl -> jcl.getJoinNamesAsString()).toList()) + "." + )); + } + else + { + exposedJoin.setJoinPath(eligibleJoinConnections.get(0).getJoinNamesAsList()); + } + } + } + } + } + } + } + catch(Exception e) + { + throw (new RuntimeException("Error enriching instance joins", e)); + } } @@ -1016,4 +1096,13 @@ public class QInstanceEnricher } } + + + /******************************************************************************* + ** + *******************************************************************************/ + public JoinGraph getJoinGraph() + { + return (this.joinGraph); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 1a211992..64f85dde 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -36,6 +36,7 @@ import java.util.function.Supplier; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface; import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; @@ -69,6 +70,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; @@ -115,17 +117,26 @@ public class QInstanceValidator return; } + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // the enricher will build a join graph (if there are any joins). we'd like to only do that // + // once, during the enrichment/validation work, so, capture it, and store it back in the instance. // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + JoinGraph joinGraph = null; try { ///////////////////////////////////////////////////////////////////////////////////////////////// // before validation, enrich the object (e.g., to fill in values that the user doesn't have to // ///////////////////////////////////////////////////////////////////////////////////////////////// // TODO - possible point of customization (use a different enricher, or none, or pass it options). - new QInstanceEnricher(qInstance).enrich(); + QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(qInstance); + qInstanceEnricher.enrich(); + joinGraph = qInstanceEnricher.getJoinGraph(); } catch(Exception e) { + System.out.println(); LOG.error("Error enriching instance prior to validation", e); + System.out.println(); throw (new QInstanceValidationException("Error enriching qInstance prior to validation.", e)); } @@ -136,7 +147,7 @@ public class QInstanceValidator { validateBackends(qInstance); validateAutomationProviders(qInstance); - validateTables(qInstance); + validateTables(qInstance, joinGraph); validateProcesses(qInstance); validateReports(qInstance); validateApps(qInstance); @@ -158,7 +169,9 @@ public class QInstanceValidator throw (new QInstanceValidationException(errors)); } - qInstance.setHasBeenValidated(new QInstanceValidationKey()); + QInstanceValidationKey validationKey = new QInstanceValidationKey(); + qInstance.setHasBeenValidated(validationKey); + qInstance.setJoinGraph(validationKey, joinGraph); } @@ -366,7 +379,7 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateTables(QInstance qInstance) + private void validateTables(QInstance qInstance, JoinGraph joinGraph) { if(assertCondition(CollectionUtils.nullSafeHasContents(qInstance.getTables()), "At least 1 table must be defined.")) { @@ -466,12 +479,61 @@ public class QInstanceValidator validateTableCacheOf(qInstance, table); validateTableRecordSecurityLocks(qInstance, table); validateTableAssociations(qInstance, table); + validateExposedJoins(qInstance, joinGraph, table); }); } } + /******************************************************************************* + ** + *******************************************************************************/ + private void validateExposedJoins(QInstance qInstance, JoinGraph joinGraph, QTableMetaData table) + { + Set joinConnectionsForTable = null; + Set usedLabels = new HashSet<>(); + Set> usedJoinPaths = new HashSet<>(); + + String tablePrefix = "Table " + table.getName() + " "; + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + String joinPrefix = tablePrefix + "exposedJoin [missingJoinTableName] "; + if(assertCondition(StringUtils.hasContent(exposedJoin.getJoinTable()), tablePrefix + "has an exposedJoin that is missing a joinTable name.")) + { + joinPrefix = tablePrefix + "exposedJoin " + exposedJoin.getJoinTable() + " "; + if(assertCondition(qInstance.getTable(exposedJoin.getJoinTable()) != null, joinPrefix + "is referencing an unrecognized table")) + { + if(assertCondition(CollectionUtils.nullSafeHasContents(exposedJoin.getJoinPath()), joinPrefix + "is missing a joinPath.")) + { + joinConnectionsForTable = Objects.requireNonNullElseGet(joinConnectionsForTable, () -> joinGraph.getJoinConnections(table.getName())); + + boolean foundJoinConnection = false; + for(JoinGraph.JoinConnectionList joinConnectionList : joinConnectionsForTable) + { + if(joinConnectionList.matchesJoinPath(exposedJoin.getJoinPath())) + { + foundJoinConnection = true; + } + } + assertCondition(foundJoinConnection, joinPrefix + "specified a joinPath [" + exposedJoin.getJoinPath() + "] which does not match a valid join connection in the instance."); + + assertCondition(!usedJoinPaths.contains(exposedJoin.getJoinPath()), tablePrefix + "has more than one join with the joinPath: " + exposedJoin.getJoinPath()); + usedJoinPaths.add(exposedJoin.getJoinPath()); + } + } + } + + if(assertCondition(StringUtils.hasContent(exposedJoin.getLabel()), joinPrefix + "is missing a label.")) + { + assertCondition(!usedLabels.contains(exposedJoin.getLabel()), tablePrefix + "has more than one join labeled: " + exposedJoin.getLabel()); + usedLabels.add(exposedJoin.getLabel()); + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountInput.java index 09ff116c..1a4863e5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountInput.java @@ -37,7 +37,8 @@ public class CountInput extends AbstractTableActionInput { private QQueryFilter filter; - private List queryJoins = null; + private List queryJoins = null; + private Boolean includeDistinctCount = false; @@ -120,4 +121,35 @@ public class CountInput extends AbstractTableActionInput return (this); } + + + /******************************************************************************* + ** Getter for includeDistinctCount + *******************************************************************************/ + public Boolean getIncludeDistinctCount() + { + return (this.includeDistinctCount); + } + + + + /******************************************************************************* + ** Setter for includeDistinctCount + *******************************************************************************/ + public void setIncludeDistinctCount(Boolean includeDistinctCount) + { + this.includeDistinctCount = includeDistinctCount; + } + + + + /******************************************************************************* + ** Fluent setter for includeDistinctCount + *******************************************************************************/ + public CountInput withIncludeDistinctCount(Boolean includeDistinctCount) + { + this.includeDistinctCount = includeDistinctCount; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountOutput.java index d3703cfa..0484a175 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountOutput.java @@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; public class CountOutput extends AbstractActionOutput { private Integer count; + private Integer distinctCount; @@ -52,4 +53,47 @@ public class CountOutput extends AbstractActionOutput { this.count = count; } + + + + /******************************************************************************* + ** Getter for distinctCount + *******************************************************************************/ + public Integer getDistinctCount() + { + return (this.distinctCount); + } + + + + /******************************************************************************* + ** Setter for distinctCount + *******************************************************************************/ + public void setDistinctCount(Integer distinctCount) + { + this.distinctCount = distinctCount; + } + + + + /******************************************************************************* + ** Fluent setter for distinctCount + *******************************************************************************/ + public CountOutput withDistinctCount(Integer distinctCount) + { + this.distinctCount = distinctCount; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for count + *******************************************************************************/ + public CountOutput withCount(Integer count) + { + this.count = count; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java index 5a66e1e3..2a52e634 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java @@ -32,12 +32,16 @@ import java.util.Objects; import java.util.Set; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.collections.MutableList; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -46,6 +50,8 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; *******************************************************************************/ public class JoinsContext { + private static final QLogger LOG = QLogger.getLogger(JoinsContext.class); + private final QInstance instance; private final String mainTableName; private final List queryJoins; @@ -65,7 +71,7 @@ public class JoinsContext { this.instance = instance; this.mainTableName = tableName; - this.queryJoins = CollectionUtils.nonNullList(queryJoins); + this.queryJoins = new MutableList<>(queryJoins); for(QueryJoin queryJoin : this.queryJoins) { @@ -123,6 +129,8 @@ public class JoinsContext ensureFilterIsRepresented(filter); + addJoinsFromExposedJoinPaths(); + /* todo!! for(QueryJoin queryJoin : queryJoins) { @@ -132,7 +140,147 @@ public class JoinsContext // addCriteriaForRecordSecurityLock(instance, session, joinTable, securityCriteria, recordSecurityLock, joinsContext, queryJoin.getJoinTableOrItsAlias()); } } - */ + */ + } + + + + /******************************************************************************* + ** If there are any joins in the context that don't have a join meta data, see + ** if we can find the JoinMetaData to use for them by looking at the main table's + ** exposed joins, and using their join paths. + *******************************************************************************/ + private void addJoinsFromExposedJoinPaths() throws QException + { + //////////////////////////////////////////////////////////////////////////////// + // do a double-loop, to avoid concurrent modification on the queryJoins list. // + // that is to say, we'll loop over that list, but possibly add things to it, // + // in which case we'll set this flag, and break the inner loop, to go again. // + //////////////////////////////////////////////////////////////////////////////// + boolean addedJoin; + do + { + addedJoin = false; + for(QueryJoin queryJoin : queryJoins) + { + ///////////////////////////////////////////////////////////////////// + // if the join has joinMetaData, then we don't need to process it. // + ///////////////////////////////////////////////////////////////////// + if(queryJoin.getJoinMetaData() == null) + { + ////////////////////////////////////////////////////////////////////// + // try to find a direct join between the main table and this table. // + // if one is found, then put it (the meta data) on the query join. // + ////////////////////////////////////////////////////////////////////// + String baseTableName = Objects.requireNonNullElse(resolveTableNameOrAliasToTableName(queryJoin.getBaseTableOrAlias()), mainTableName); + QJoinMetaData found = findJoinMetaData(instance, baseTableName, queryJoin.getJoinTable()); + if(found != null) + { + queryJoin.setJoinMetaData(found); + } + else + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else, the join must be indirect - so look for an exposedJoin that will have a joinPath that will connect us // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + LOG.debug("Looking for an exposed join...", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable())); + + QTableMetaData mainTable = instance.getTable(mainTableName); + boolean addedAnyQueryJoins = false; + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(mainTable.getExposedJoins())) + { + if(queryJoin.getJoinTable().equals(exposedJoin.getJoinTable())) + { + LOG.debug("Found an exposed join", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()), logPair("joinPath", exposedJoin.getJoinPath())); + + ///////////////////////////////////////////////////////////////////////////////////// + // loop backward through the join path (from the joinTable back to the main table) // + // adding joins to the table (if they aren't already in the query) // + ///////////////////////////////////////////////////////////////////////////////////// + String tmpTable = queryJoin.getJoinTable(); + for(int i = exposedJoin.getJoinPath().size() - 1; i >= 0; i--) + { + String joinName = exposedJoin.getJoinPath().get(i); + QJoinMetaData joinToAdd = instance.getJoin(joinName); + + ///////////////////////////////////////////////////////////////////////////// + // get the name from the opposite side of the join (flipping it if needed) // + ///////////////////////////////////////////////////////////////////////////// + String nextTable; + if(joinToAdd.getRightTable().equals(tmpTable)) + { + nextTable = joinToAdd.getLeftTable(); + } + else + { + nextTable = joinToAdd.getRightTable(); + joinToAdd = joinToAdd.flip(); + } + + if(doesJoinNeedAddedToQuery(joinName)) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if this is the last element in the joinPath, then we want to set this joinMetaData on the outer queryJoin // + // - else, we need to add a new queryJoin to this context // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(i == exposedJoin.getJoinPath().size() - 1) + { + if(queryJoin.getBaseTableOrAlias() == null) + { + queryJoin.setBaseTableOrAlias(nextTable); + } + queryJoin.setJoinMetaData(joinToAdd); + } + else + { + QueryJoin queryJoinToAdd = makeQueryJoinFromJoinAndTableNames(nextTable, tmpTable, joinToAdd); + queryJoinToAdd.setType(queryJoin.getType()); + addedAnyQueryJoins = true; + this.queryJoins.add(queryJoinToAdd); // todo something else with aliases? probably. + processQueryJoin(queryJoinToAdd); + } + } + + tmpTable = nextTable; + } + } + } + + /////////////////////////////////////////////////////////////////////////////////////////////////// + // break the inner loop (it would fail due to a concurrent modification), but continue the outer // + /////////////////////////////////////////////////////////////////////////////////////////////////// + if(addedAnyQueryJoins) + { + addedJoin = true; + break; + } + } + } + } + } + while(addedJoin); + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean doesJoinNeedAddedToQuery(String joinName) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // look at all queryJoins already in context - if any have this join's name, then we don't need this join... // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QueryJoin queryJoin : queryJoins) + { + if(queryJoin.getJoinMetaData() != null && queryJoin.getJoinMetaData().getName().equals(joinName)) + { + return (false); + } + } + + return (true); } @@ -256,34 +404,54 @@ public class JoinsContext { if(!aliasToTableNameMap.containsKey(filterTable) && !Objects.equals(mainTableName, filterTable)) { + boolean found = false; for(QJoinMetaData join : CollectionUtils.nonNullMap(QContext.getQInstance().getJoins()).values()) { - QueryJoin queryJoin = null; - if(join.getLeftTable().equals(mainTableName) && join.getRightTable().equals(filterTable)) - { - queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER); - } - else - { - join = join.flip(); - if(join.getLeftTable().equals(mainTableName) && join.getRightTable().equals(filterTable)) - { - queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER); - } - } - + QueryJoin queryJoin = makeQueryJoinFromJoinAndTableNames(mainTableName, filterTable, join); if(queryJoin != null) { this.queryJoins.add(queryJoin); // todo something else with aliases? probably. processQueryJoin(queryJoin); + found = true; + break; } } + + if(!found) + { + QueryJoin queryJoin = new QueryJoin().withJoinTable(filterTable).withType(QueryJoin.Type.INNER); + this.queryJoins.add(queryJoin); // todo something else with aliases? probably. + processQueryJoin(queryJoin); + } } } } + /******************************************************************************* + ** + *******************************************************************************/ + private QueryJoin makeQueryJoinFromJoinAndTableNames(String tableA, String tableB, QJoinMetaData join) + { + QueryJoin queryJoin = null; + if(join.getLeftTable().equals(tableA) && join.getRightTable().equals(tableB)) + { + queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER); + } + else + { + join = join.flip(); + if(join.getLeftTable().equals(tableA) && join.getRightTable().equals(tableB)) + { + queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER); + } + } + return queryJoin; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java index f89d23e4..6ce122bb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java @@ -47,6 +47,13 @@ public class QQueryFilter implements Serializable, Cloneable private BooleanOperator booleanOperator = BooleanOperator.AND; private List subFilters = new ArrayList<>(); + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // skip & limit are meant to only apply to QueryAction (at least at the initial time they are added here) // + // e.g., they are ignored in CountAction, AggregateAction, etc, where their meanings may be less obvious // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + private Integer skip; + private Integer limit; + /******************************************************************************* @@ -398,4 +405,66 @@ public class QQueryFilter implements Serializable, Cloneable } } + + + /******************************************************************************* + ** Getter for skip + *******************************************************************************/ + public Integer getSkip() + { + return (this.skip); + } + + + + /******************************************************************************* + ** Setter for skip + *******************************************************************************/ + public void setSkip(Integer skip) + { + this.skip = skip; + } + + + + /******************************************************************************* + ** Fluent setter for skip + *******************************************************************************/ + public QQueryFilter withSkip(Integer skip) + { + this.skip = skip; + return (this); + } + + + + /******************************************************************************* + ** Getter for limit + *******************************************************************************/ + public Integer getLimit() + { + return (this.limit); + } + + + + /******************************************************************************* + ** Setter for limit + *******************************************************************************/ + public void setLimit(Integer limit) + { + this.limit = limit; + } + + + + /******************************************************************************* + ** Fluent setter for limit + *******************************************************************************/ + public QQueryFilter withLimit(Integer limit) + { + this.limit = limit; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java index 76c4101f..6cb2c556 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java @@ -39,8 +39,6 @@ public class QueryInput extends AbstractTableActionInput { private QBackendTransaction transaction; private QQueryFilter filter; - private Integer skip; - private Integer limit; private RecordPipe recordPipe; @@ -57,7 +55,8 @@ public class QueryInput extends AbstractTableActionInput ///////////////////////////////////////////////////////////////////////////////////////// private Set fieldsToTranslatePossibleValues; - private List queryJoins = null; + private List queryJoins = null; + private boolean selectDistinct = false; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if you say you want to includeAssociations, you can limit which ones by passing them in associationNamesToInclude. // @@ -100,50 +99,6 @@ public class QueryInput extends AbstractTableActionInput - /******************************************************************************* - ** Getter for skip - ** - *******************************************************************************/ - public Integer getSkip() - { - return skip; - } - - - - /******************************************************************************* - ** Setter for skip - ** - *******************************************************************************/ - public void setSkip(Integer skip) - { - this.skip = skip; - } - - - - /******************************************************************************* - ** Getter for limit - ** - *******************************************************************************/ - public Integer getLimit() - { - return limit; - } - - - - /******************************************************************************* - ** Setter for limit - ** - *******************************************************************************/ - public void setLimit(Integer limit) - { - this.limit = limit; - } - - - /******************************************************************************* ** Getter for recordPipe ** @@ -361,28 +316,6 @@ public class QueryInput extends AbstractTableActionInput - /******************************************************************************* - ** Fluent setter for skip - *******************************************************************************/ - public QueryInput withSkip(Integer skip) - { - this.skip = skip; - return (this); - } - - - - /******************************************************************************* - ** Fluent setter for limit - *******************************************************************************/ - public QueryInput withLimit(Integer limit) - { - this.limit = limit; - return (this); - } - - - /******************************************************************************* ** Fluent setter for recordPipe *******************************************************************************/ @@ -561,4 +494,35 @@ public class QueryInput extends AbstractTableActionInput return (this); } + + + /******************************************************************************* + ** Getter for selectDistinct + *******************************************************************************/ + public boolean getSelectDistinct() + { + return (this.selectDistinct); + } + + + + /******************************************************************************* + ** Setter for selectDistinct + *******************************************************************************/ + public void setSelectDistinct(boolean selectDistinct) + { + this.selectDistinct = selectDistinct; + } + + + + /******************************************************************************* + ** Fluent setter for selectDistinct + *******************************************************************************/ + public QueryInput withSelectDistinct(boolean selectDistinct) + { + this.selectDistinct = selectDistinct; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index be3b7074..0b9f45ee 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph; import com.kingsrook.qqq.backend.core.actions.metadata.MetaDataAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.instances.QInstanceValidationKey; @@ -106,6 +107,8 @@ public class QInstance private Map memoizedTablePaths = new HashMap<>(); private Map memoizedProcessPaths = new HashMap<>(); + private JoinGraph joinGraph; + /******************************************************************************* @@ -1136,4 +1139,30 @@ public class QInstance this.middlewareMetaData.put(middlewareMetaData.getType(), middlewareMetaData); return (this); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public JoinGraph getJoinGraph() + { + return (this.joinGraph); + } + + + + /******************************************************************************* + ** Only the validation (and enrichment) code should set the instance's joinGraph + ** so, we take a package-only-constructable validation key as a param along with + ** the joinGraph - and we throw IllegalArgumentException if a non-null key is given. + *******************************************************************************/ + public void setJoinGraph(QInstanceValidationKey key, JoinGraph joinGraph) throws IllegalArgumentException + { + if(key == null) + { + throw (new IllegalArgumentException("A ValidationKey must be provided")); + } + this.joinGraph = joinGraph; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/WidgetQueryField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/WidgetQueryField.java index f5997b6d..197ce195 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/WidgetQueryField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/WidgetQueryField.java @@ -62,8 +62,9 @@ public class WidgetQueryField extends AbstractWidgetValueSourceWithFilter { QueryInput queryInput = new QueryInput(); queryInput.setTableName(tableName); - queryInput.setFilter(getEffectiveFilter(input)); - queryInput.setLimit(1); + QQueryFilter effectiveFilter = getEffectiveFilter(input); + queryInput.setFilter(effectiveFilter); + effectiveFilter.setLimit(1); QueryOutput queryOutput = new QueryAction().execute(queryInput); if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords())) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendExposedJoin.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendExposedJoin.java new file mode 100644 index 00000000..f89816c8 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendExposedJoin.java @@ -0,0 +1,178 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.frontend; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; + + +/******************************************************************************* + ** Version of an ExposedJoin for a frontend to see + *******************************************************************************/ +public class QFrontendExposedJoin +{ + private String label; + private Boolean isMany; + private QFrontendTableMetaData joinTable; + private List joinPath; + + + + /******************************************************************************* + ** Getter for label + *******************************************************************************/ + public String getLabel() + { + return (this.label); + } + + + + /******************************************************************************* + ** Setter for label + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + *******************************************************************************/ + public QFrontendExposedJoin withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for joinTable + *******************************************************************************/ + public QFrontendTableMetaData getJoinTable() + { + return (this.joinTable); + } + + + + /******************************************************************************* + ** Setter for joinTable + *******************************************************************************/ + public void setJoinTable(QFrontendTableMetaData joinTable) + { + this.joinTable = joinTable; + } + + + + /******************************************************************************* + ** Fluent setter for joinTable + *******************************************************************************/ + public QFrontendExposedJoin withJoinTable(QFrontendTableMetaData joinTable) + { + this.joinTable = joinTable; + return (this); + } + + + + /******************************************************************************* + ** Getter for joinPath + *******************************************************************************/ + public List getJoinPath() + { + return (this.joinPath); + } + + + + /******************************************************************************* + ** Setter for joinPath + *******************************************************************************/ + public void setJoinPath(List joinPath) + { + this.joinPath = joinPath; + } + + + + /******************************************************************************* + ** Fluent setter for joinPath + *******************************************************************************/ + public QFrontendExposedJoin withJoinPath(List joinPath) + { + this.joinPath = joinPath; + return (this); + } + + + + /******************************************************************************* + ** Add one join to the join path in here + *******************************************************************************/ + public void addJoin(QJoinMetaData join) + { + if(this.joinPath == null) + { + this.joinPath = new ArrayList<>(); + } + this.joinPath.add(join); + } + + + + /******************************************************************************* + ** Getter for isMany + *******************************************************************************/ + public Boolean getIsMany() + { + return (this.isMany); + } + + + + /******************************************************************************* + ** Setter for isMany + *******************************************************************************/ + public void setIsMany(Boolean isMany) + { + this.isMany = isMany; + } + + + + /******************************************************************************* + ** Fluent setter for isMany + *******************************************************************************/ + public QFrontendExposedJoin withIsMany(Boolean isMany) + { + this.isMany = isMany; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java index c53cc933..199d42a6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.frontend; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -32,12 +33,16 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; /******************************************************************************* @@ -58,6 +63,8 @@ public class QFrontendTableMetaData private Map fields; private List sections; + private List exposedJoins; + private Set capabilities; private boolean readPermission; @@ -74,7 +81,7 @@ public class QFrontendTableMetaData /******************************************************************************* ** *******************************************************************************/ - public QFrontendTableMetaData(AbstractActionInput actionInput, QBackendMetaData backendForTable, QTableMetaData tableMetaData, boolean includeFields) + public QFrontendTableMetaData(AbstractActionInput actionInput, QBackendMetaData backendForTable, QTableMetaData tableMetaData, boolean includeFields, boolean includeJoins) { this.name = tableMetaData.getName(); this.label = tableMetaData.getLabel(); @@ -92,6 +99,27 @@ public class QFrontendTableMetaData this.sections = tableMetaData.getSections(); } + if(includeJoins) + { + QInstance qInstance = QContext.getQInstance(); + + this.exposedJoins = new ArrayList<>(); + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(tableMetaData.getExposedJoins())) + { + QFrontendExposedJoin frontendExposedJoin = new QFrontendExposedJoin(); + this.exposedJoins.add(frontendExposedJoin); + + QTableMetaData joinTable = qInstance.getTable(exposedJoin.getJoinTable()); + frontendExposedJoin.setLabel(exposedJoin.getLabel()); + frontendExposedJoin.setIsMany(exposedJoin.getIsMany()); + frontendExposedJoin.setJoinTable(new QFrontendTableMetaData(actionInput, backendForTable, joinTable, includeFields, false)); + for(String joinName : exposedJoin.getJoinPath()) + { + frontendExposedJoin.addJoin(qInstance.getJoin(joinName)); + } + } + } + if(tableMetaData.getIcon() != null) { this.iconName = tableMetaData.getIcon().getName(); @@ -259,4 +287,15 @@ public class QFrontendTableMetaData { return deletePermission; } + + + + /******************************************************************************* + ** Getter for exposedJoins + ** + *******************************************************************************/ + public List getExposedJoins() + { + return exposedJoins; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoin.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoin.java new file mode 100644 index 00000000..f66b1b23 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoin.java @@ -0,0 +1,208 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ExposedJoin +{ + private static final QLogger LOG = QLogger.getLogger(ExposedJoin.class); + + private String label; + private String joinTable; + private List joinPath; + + ////////////////////////////////////////////////////////////////// + // no setter for this - derive it the first time it's requested // + ////////////////////////////////////////////////////////////////// + private Boolean isMany = null; + + + + /******************************************************************************* + ** Getter for label + *******************************************************************************/ + public String getLabel() + { + return (this.label); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Boolean getIsMany() + { + if(isMany == null) + { + if(CollectionUtils.nullSafeHasContents(joinPath)) + { + try + { + QInstance qInstance = QContext.getQInstance(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // loop backward through the joinPath, starting at the join table (since we don't know the table that this exposedJoin is attached to!) // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + String currentTable = joinTable; + for(int i = joinPath.size() - 1; i >= 0; i--) + { + String joinName = joinPath.get(i); + QJoinMetaData join = qInstance.getJoin(joinName); + if(join.getRightTable().equals(currentTable)) + { + currentTable = join.getLeftTable(); + if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_MANY)) + { + isMany = true; + break; + } + } + else if(join.getLeftTable().equals(currentTable)) + { + currentTable = join.getRightTable(); + if(join.getType().equals(JoinType.MANY_TO_ONE) || join.getType().equals(JoinType.MANY_TO_MANY)) + { + isMany = true; + break; + } + } + else + { + throw (new IllegalStateException("Current join table [" + currentTable + "] in path traversal was not found at element [" + joinName + "]")); + } + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we successfully got through the loop, and never found a reason to mark this join as "many", then it must not be, so set isMany to false // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(isMany == null) + { + isMany = false; + } + } + catch(Exception e) + { + LOG.warn("Error deriving if ExposedJoin through [" + joinPath + "] to [" + joinTable + "] isMany", e); + } + } + } + + return (isMany); + } + + + + /******************************************************************************* + ** Setter for label + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + *******************************************************************************/ + public ExposedJoin withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for joinTable + *******************************************************************************/ + public String getJoinTable() + { + return (this.joinTable); + } + + + + /******************************************************************************* + ** Setter for joinTable + *******************************************************************************/ + public void setJoinTable(String joinTable) + { + this.joinTable = joinTable; + } + + + + /******************************************************************************* + ** Fluent setter for joinTable + *******************************************************************************/ + public ExposedJoin withJoinTable(String joinTable) + { + this.joinTable = joinTable; + return (this); + } + + + + /******************************************************************************* + ** Getter for joinPath + *******************************************************************************/ + public List getJoinPath() + { + return (this.joinPath); + } + + + + /******************************************************************************* + ** Setter for joinPath + *******************************************************************************/ + public void setJoinPath(List joinPath) + { + this.joinPath = joinPath; + } + + + + /******************************************************************************* + ** Fluent setter for joinPath + *******************************************************************************/ + public ExposedJoin withJoinPath(List joinPath) + { + this.joinPath = joinPath; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index 77249eef..2fa71424 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -101,6 +101,8 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData private Map middlewareMetaData; + private List exposedJoins; + /******************************************************************************* @@ -1296,4 +1298,51 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData { qInstance.addTable(this); } + + + + /******************************************************************************* + ** Getter for exposedJoins + *******************************************************************************/ + public List getExposedJoins() + { + return (this.exposedJoins); + } + + + + /******************************************************************************* + ** Setter for exposedJoins + *******************************************************************************/ + public void setExposedJoins(List exposedJoins) + { + this.exposedJoins = exposedJoins; + } + + + + /******************************************************************************* + ** Fluent setter for exposedJoins + *******************************************************************************/ + public QTableMetaData withExposedJoins(List exposedJoins) + { + this.exposedJoins = exposedJoins; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for exposedJoins + *******************************************************************************/ + public QTableMetaData withExposedJoin(ExposedJoin exposedJoin) + { + if(this.exposedJoins == null) + { + this.exposedJoins = new ArrayList<>(); + } + this.exposedJoins.add(exposedJoin); + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryAction.java index 039e0123..c04d1dd1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryAction.java @@ -69,7 +69,7 @@ public class EnumerationQueryAction implements QueryInterface } BackendQueryFilterUtils.sortRecordList(queryInput.getFilter(), recordList); - recordList = BackendQueryFilterUtils.applySkipAndLimit(queryInput, recordList); + recordList = BackendQueryFilterUtils.applySkipAndLimit(queryInput.getFilter(), recordList); QueryOutput queryOutput = new QueryOutput(queryInput); queryOutput.addRecords(recordList); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java index 6cc00b8f..b25cac78 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java @@ -38,13 +38,16 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -181,7 +184,7 @@ public class MemoryRecordStore } BackendQueryFilterUtils.sortRecordList(input.getFilter(), records); - records = BackendQueryFilterUtils.applySkipAndLimit(input, records); + records = BackendQueryFilterUtils.applySkipAndLimit(input.getFilter(), records); return (records); } @@ -191,8 +194,11 @@ public class MemoryRecordStore /******************************************************************************* ** *******************************************************************************/ - private Collection buildJoinCrossProduct(QueryInput input) + private Collection buildJoinCrossProduct(QueryInput input) throws QException { + QInstance qInstance = QContext.getQInstance(); + JoinsContext joinsContext = new JoinsContext(qInstance, input.getTableName(), input.getQueryJoins(), input.getFilter()); + List crossProduct = new ArrayList<>(); QTableMetaData leftTable = input.getTable(); for(QRecord record : getTableData(leftTable).values()) @@ -204,16 +210,26 @@ public class MemoryRecordStore for(QueryJoin queryJoin : input.getQueryJoins()) { - QTableMetaData nextTable = QContext.getQInstance().getTable(queryJoin.getJoinTable()); + QTableMetaData nextTable = qInstance.getTable(queryJoin.getJoinTable()); Collection nextTableRecords = getTableData(nextTable).values(); + QJoinMetaData joinMetaData = Objects.requireNonNullElseGet(queryJoin.getJoinMetaData(), () -> + { + QJoinMetaData found = joinsContext.findJoinMetaData(qInstance, input.getTableName(), queryJoin.getJoinTable()); + if(found == null) + { + throw (new RuntimeException("Could not find a join between tables [" + input.getTableName() + "][" + queryJoin.getJoinTable() + "]")); + } + return (found); + }); + List nextLevelProduct = new ArrayList<>(); for(QRecord productRecord : crossProduct) { boolean matchFound = false; for(QRecord nextTableRecord : nextTableRecords) { - if(joinMatches(productRecord, nextTableRecord, queryJoin)) + if(joinMatches(productRecord, nextTableRecord, queryJoin, joinMetaData)) { QRecord joinRecord = new QRecord(productRecord); addRecordToProduct(joinRecord, nextTableRecord, queryJoin.getJoinTableOrItsAlias()); @@ -239,9 +255,9 @@ public class MemoryRecordStore /******************************************************************************* ** *******************************************************************************/ - private boolean joinMatches(QRecord productRecord, QRecord nextTableRecord, QueryJoin queryJoin) + private boolean joinMatches(QRecord productRecord, QRecord nextTableRecord, QueryJoin queryJoin, QJoinMetaData joinMetaData) { - for(JoinOn joinOn : queryJoin.getJoinMetaData().getJoinOns()) + for(JoinOn joinOn : joinMetaData.getJoinOns()) { Serializable leftValue = productRecord.getValues().containsKey(queryJoin.getBaseTableOrAlias() + "." + joinOn.getLeftField()) ? productRecord.getValue(queryJoin.getBaseTableOrAlias() + "." + joinOn.getLeftField()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java index f99b6efb..9fc87831 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java @@ -27,7 +27,6 @@ import java.math.BigDecimal; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.Month; -import java.util.Objects; import java.util.UUID; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -36,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; /******************************************************************************* @@ -59,7 +59,8 @@ public class MockQueryAction implements QueryInterface QueryOutput queryOutput = new QueryOutput(queryInput); - int rows = Objects.requireNonNullElse(queryInput.getLimit(), 1); + @SuppressWarnings("UnnecessaryUnboxing") // force an un-boxing, to force an NPE if it's null, to get to the "else 1" + int rows = ObjectUtils.tryElse(() -> queryInput.getFilter().getLimit().intValue(), 1); for(int i = 0; i < rows; i++) { QRecord record = new QRecord(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java index abf691d8..e05d0bbd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java @@ -33,7 +33,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -129,42 +128,42 @@ public class BackendQueryFilterUtils public static boolean doesCriteriaMatch(QFilterCriteria criterion, String fieldName, Serializable value) { boolean criterionMatches = switch(criterion.getOperator()) + { + case EQUALS -> testEquals(criterion, value); + case NOT_EQUALS -> !testEquals(criterion, value); + case IN -> testIn(criterion, value); + case NOT_IN -> !testIn(criterion, value); + case IS_BLANK -> testBlank(criterion, value); + case IS_NOT_BLANK -> !testBlank(criterion, value); + case CONTAINS -> testContains(criterion, fieldName, value); + case NOT_CONTAINS -> !testContains(criterion, fieldName, value); + case IS_NULL_OR_IN -> testBlank(criterion, value) || testIn(criterion, value); + case LIKE -> testLike(criterion, fieldName, value); + case NOT_LIKE -> !testLike(criterion, fieldName, value); + case STARTS_WITH -> testStartsWith(criterion, fieldName, value); + case NOT_STARTS_WITH -> !testStartsWith(criterion, fieldName, value); + case ENDS_WITH -> testEndsWith(criterion, fieldName, value); + case NOT_ENDS_WITH -> !testEndsWith(criterion, fieldName, value); + case GREATER_THAN -> testGreaterThan(criterion, value); + case GREATER_THAN_OR_EQUALS -> testGreaterThan(criterion, value) || testEquals(criterion, value); + case LESS_THAN -> !testGreaterThan(criterion, value) && !testEquals(criterion, value); + case LESS_THAN_OR_EQUALS -> !testGreaterThan(criterion, value); + case BETWEEN -> { - case EQUALS -> testEquals(criterion, value); - case NOT_EQUALS -> !testEquals(criterion, value); - case IN -> testIn(criterion, value); - case NOT_IN -> !testIn(criterion, value); - case IS_BLANK -> testBlank(criterion, value); - case IS_NOT_BLANK -> !testBlank(criterion, value); - case CONTAINS -> testContains(criterion, fieldName, value); - case NOT_CONTAINS -> !testContains(criterion, fieldName, value); - case IS_NULL_OR_IN -> testBlank(criterion, value) || testIn(criterion, value); - case LIKE -> testLike(criterion, fieldName, value); - case NOT_LIKE -> !testLike(criterion, fieldName, value); - case STARTS_WITH -> testStartsWith(criterion, fieldName, value); - case NOT_STARTS_WITH -> !testStartsWith(criterion, fieldName, value); - case ENDS_WITH -> testEndsWith(criterion, fieldName, value); - case NOT_ENDS_WITH -> !testEndsWith(criterion, fieldName, value); - case GREATER_THAN -> testGreaterThan(criterion, value); - case GREATER_THAN_OR_EQUALS -> testGreaterThan(criterion, value) || testEquals(criterion, value); - case LESS_THAN -> !testGreaterThan(criterion, value) && !testEquals(criterion, value); - case LESS_THAN_OR_EQUALS -> !testGreaterThan(criterion, value); - case BETWEEN -> - { - QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues()); - QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues())); - criteria1.getValues().remove(0); - yield (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value)); - } - case NOT_BETWEEN -> - { - QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues()); - QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues())); - criteria1.getValues().remove(0); - boolean between = (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value)); - yield !between; - } - }; + QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues()); + QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues())); + criteria1.getValues().remove(0); + yield (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value)); + } + case NOT_BETWEEN -> + { + QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues()); + QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues())); + criteria1.getValues().remove(0); + boolean between = (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value)); + yield !between; + } + }; return criterionMatches; } @@ -524,9 +523,14 @@ public class BackendQueryFilterUtils /******************************************************************************* ** Apply skip & limit attributes from queryInput to a list of records. *******************************************************************************/ - public static List applySkipAndLimit(QueryInput queryInput, List recordList) + public static List applySkipAndLimit(QQueryFilter queryFilter, List recordList) { - Integer skip = queryInput.getSkip(); + if(queryFilter == null) + { + return (recordList); + } + + Integer skip = queryFilter.getSkip(); if(skip != null && skip > 0) { if(skip < recordList.size()) @@ -539,7 +543,7 @@ public class BackendQueryFilterUtils } } - Integer limit = queryInput.getLimit(); + Integer limit = queryFilter.getLimit(); if(limit != null && limit >= 0 && limit < recordList.size()) { recordList = recordList.subList(0, limit); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java index f1f9c873..b3dddc99 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java @@ -211,14 +211,11 @@ public class BulkEditTransformStep extends AbstractTransformStep @Override public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) { - if(isForResultScreen) - { - okSummary.setMessage(tableLabel + " records were edited."); - } - else - { - okSummary.setMessage(tableLabel + " records will be edited."); - } + okSummary.setSingularFutureMessage(tableLabel + " record will be edited."); + okSummary.setPluralFutureMessage(tableLabel + " records will be edited."); + okSummary.setSingularPastMessage(tableLabel + " record was edited."); + okSummary.setPluralPastMessage(tableLabel + " records were edited."); + okSummary.pickMessage(isForResultScreen); ArrayList rs = new ArrayList<>(); rs.add(okSummary); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java index ee23b94f..8f38b8fd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java @@ -29,6 +29,7 @@ import java.util.List; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; @@ -52,6 +53,8 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; *******************************************************************************/ public class ExtractViaQueryStep extends AbstractExtractStep { + private static final QLogger LOG = QLogger.getLogger(ExtractViaQueryStep.class); + public static final String FIELD_SOURCE_TABLE = "sourceTable"; private QQueryFilter queryFilter; @@ -77,11 +80,33 @@ public class ExtractViaQueryStep extends AbstractExtractStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { + ////////////////////////////////////////////////////////////////// + // clone the filter, since we're going to edit it (set a limit) // + ////////////////////////////////////////////////////////////////// + QQueryFilter filterClone = queryFilter.clone(); + + ////////////////////////////////////////////////////////////////////////////////////////////// + // if there's a limit in the extract step (e.g., the 20-record limit on the preview screen) // + // then set that limit in the filter - UNLESS - there's already a limit in the filter for // + // a smaller number of records. // + ////////////////////////////////////////////////////////////////////////////////////////////// + if(getLimit() != null) + { + if(filterClone.getLimit() != null && filterClone.getLimit() < getLimit()) + { + LOG.trace("Using filter's limit [" + filterClone.getLimit() + "] rather than step's limit [" + getLimit() + "]"); + } + else + { + filterClone.setLimit(getLimit()); + } + } + QueryInput queryInput = new QueryInput(); queryInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE)); - queryInput.setFilter(queryFilter); + queryInput.setFilter(filterClone); + queryInput.setSelectDistinct(true); queryInput.setRecordPipe(getRecordPipe()); - queryInput.setLimit(getLimit()); queryInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback()); customizeInputPreQuery(queryInput); @@ -114,8 +139,20 @@ public class ExtractViaQueryStep extends AbstractExtractStep CountInput countInput = new CountInput(); countInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE)); countInput.setFilter(queryFilter); + countInput.setIncludeDistinctCount(true); CountOutput countOutput = new CountAction().execute(countInput); - return (countOutput.getCount()); + Integer count = countOutput.getDistinctCount(); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // in case the filter we're running has a limit, but the count found more than that limit, // + // well then, just return that limit - as the process won't run on more rows than that. // + ///////////////////////////////////////////////////////////////////////////////////////////// + if(count != null & queryFilter.getLimit() != null && count > queryFilter.getLimit()) + { + count = queryFilter.getLimit(); + } + + return count; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java index db523e40..2efb278e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java @@ -85,8 +85,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep queryInput.setFilter(new QQueryFilter() .withCriteria(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, List.of(script.getValue("id")))) .withOrderBy(new QFilterOrderBy("sequenceNo", false)) - ); - queryInput.setLimit(1); + .withLimit(1)); QueryOutput queryOutput = new QueryAction().execute(queryInput); if(!queryOutput.getRecords().isEmpty()) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java index 4cde210b..98a41a4f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java @@ -209,8 +209,7 @@ public class GeneralProcessUtils QueryInput queryInput = new QueryInput(); queryInput.setTableName(tableName); - queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, fieldValue))); - queryInput.setLimit(1); + queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, fieldValue)).withLimit(1)); QueryOutput queryOutput = new QueryAction().execute(queryInput); return (queryOutput.getRecords().stream().findFirst()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableList.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableList.java index b43fd241..78084bbc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableList.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableList.java @@ -27,6 +27,7 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.ListIterator; +import java.util.Objects; import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.utils.lambdas.VoidVoidMethod; @@ -38,8 +39,8 @@ import com.kingsrook.qqq.backend.core.utils.lambdas.VoidVoidMethod; *******************************************************************************/ public class MutableList implements List { - private List sourceList; - private Class> mutableTypeIfNeeded; + private List sourceList; + private Supplier> supplierIfNeeded; @@ -49,7 +50,7 @@ public class MutableList implements List *******************************************************************************/ public MutableList(List sourceList) { - this(sourceList, (Class) ArrayList.class); + this(sourceList, ArrayList::new); } @@ -58,10 +59,20 @@ public class MutableList implements List ** Constructor ** *******************************************************************************/ - public MutableList(List sourceList, Class> mutableTypeIfNeeded) + public MutableList(List sourceList, Supplier> supplierIfNeeded) { - this.sourceList = sourceList; - this.mutableTypeIfNeeded = mutableTypeIfNeeded; + this.sourceList = Objects.requireNonNullElseGet(sourceList, supplierIfNeeded); + this.supplierIfNeeded = supplierIfNeeded; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + List getUnderlyingList() + { + return (sourceList); } @@ -73,13 +84,13 @@ public class MutableList implements List { try { - List replacementList = mutableTypeIfNeeded.getConstructor().newInstance(); + List replacementList = supplierIfNeeded.get(); replacementList.addAll(sourceList); sourceList = replacementList; } catch(Exception e) { - throw (new IllegalStateException("The mutable type provided for this MutableList [" + mutableTypeIfNeeded.getName() + "] could not be instantiated.")); + throw (new IllegalStateException("Error getting from the supplier provided for this MutableList.", e)); } } @@ -88,7 +99,7 @@ public class MutableList implements List /******************************************************************************* ** *******************************************************************************/ - private T doMutableOperationForValue(Supplier supplier) + private V doMutableOperationForValue(Supplier supplier) { try { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMap.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMap.java index 79102a07..87789756 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMap.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMap.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.utils.collections; import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.utils.lambdas.VoidVoidMethod; @@ -37,8 +38,8 @@ import com.kingsrook.qqq.backend.core.utils.lambdas.VoidVoidMethod; *******************************************************************************/ public class MutableMap implements Map { - private Map sourceMap; - private Class> mutableTypeIfNeeded; + private Map sourceMap; + private Supplier> supplierIfNeeded; @@ -48,7 +49,7 @@ public class MutableMap implements Map *******************************************************************************/ public MutableMap(Map sourceMap) { - this(sourceMap, (Class) HashMap.class); + this(sourceMap, HashMap::new); } @@ -57,10 +58,20 @@ public class MutableMap implements Map ** Constructor ** *******************************************************************************/ - public MutableMap(Map sourceMap, Class> mutableTypeIfNeeded) + public MutableMap(Map sourceMap, Supplier> supplierIfNeeded) { - this.sourceMap = sourceMap; - this.mutableTypeIfNeeded = mutableTypeIfNeeded; + this.sourceMap = Objects.requireNonNullElseGet(sourceMap, supplierIfNeeded); + this.supplierIfNeeded = supplierIfNeeded; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + Map getUnderlyingMap() + { + return (sourceMap); } @@ -72,13 +83,13 @@ public class MutableMap implements Map { try { - Map replacementMap = mutableTypeIfNeeded.getConstructor().newInstance(); + Map replacementMap = supplierIfNeeded.get(); replacementMap.putAll(sourceMap); sourceMap = replacementMap; } catch(Exception e) { - throw (new IllegalStateException("The mutable type provided for this MutableMap [" + mutableTypeIfNeeded.getName() + "] could not be instantiated.")); + throw (new IllegalStateException("Error getting from the supplier provided for this MutableMap.", e)); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java index a4fccc5c..9ff5f7de 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java @@ -22,10 +22,12 @@ package com.kingsrook.qqq.backend.core.actions.reporting; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.BaseTest; @@ -36,16 +38,21 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; import org.apache.commons.io.FileUtils; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -117,6 +124,68 @@ class ExportActionTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testJoins() throws QException, IOException + { + QInstance qInstance = QContext.getQInstance(); + QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true); + + TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_ORDER), List.of( + new QRecord().withValue("id", 1).withValue("orderNo", "ORD1").withValue("storeId", 1), + new QRecord().withValue("id", 2).withValue("orderNo", "ORD2").withValue("storeId", 1) + )); + + TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_LINE_ITEM), List.of( + new QRecord().withValue("id", 1).withValue("orderId", 1).withValue("sku", "A").withValue("quantity", 10), + new QRecord().withValue("id", 2).withValue("orderId", 1).withValue("sku", "B").withValue("quantity", 15), + new QRecord().withValue("id", 3).withValue("orderId", 2).withValue("sku", "A").withValue("quantity", 20) + )); + + ExportInput exportInput = new ExportInput(); + exportInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + exportInput.setReportFormat(ReportFormat.CSV); + ByteArrayOutputStream reportOutputStream = new ByteArrayOutputStream(); + exportInput.setReportOutputStream(reportOutputStream); + exportInput.setQueryFilter(new QQueryFilter()); + exportInput.setFieldNames(List.of("id", "orderNo", "storeId", "orderLine.id", "orderLine.sku", "orderLine.quantity")); + // exportInput.setFieldNames(List.of("id", "orderNo", "storeId")); + new ExportAction().execute(exportInput); + + String csv = reportOutputStream.toString(StandardCharsets.UTF_8); + CSVParser parse = CSVParser.parse(csv, CSVFormat.DEFAULT.withFirstRecordAsHeader()); + Iterator csvRecordIterator = parse.iterator(); + assertFalse(parse.getHeaderMap().isEmpty()); + assertTrue(parse.getHeaderMap().containsKey("Id")); + assertTrue(parse.getHeaderMap().containsKey("Order Line: Id")); + assertTrue(parse.getHeaderMap().containsKey("Order Line: SKU")); + + CSVRecord csvRecord = csvRecordIterator.next(); + assertEquals("1", csvRecord.get("Id")); + assertEquals("1", csvRecord.get("Order Line: Id")); + assertEquals("A", csvRecord.get("Order Line: SKU")); + assertEquals("10", csvRecord.get("Order Line: Quantity")); + + csvRecord = csvRecordIterator.next(); + assertEquals("1", csvRecord.get("Id")); + assertEquals("2", csvRecord.get("Order Line: Id")); + assertEquals("B", csvRecord.get("Order Line: SKU")); + assertEquals("15", csvRecord.get("Order Line: Quantity")); + + csvRecord = csvRecordIterator.next(); + assertEquals("2", csvRecord.get("Id")); + assertEquals("3", csvRecord.get("Order Line: Id")); + assertEquals("A", csvRecord.get("Order Line: SKU")); + assertEquals("20", csvRecord.get("Order Line: Quantity")); + + assertFalse(csvRecordIterator.hasNext()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java index ce9ffe45..4deb3559 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java @@ -32,7 +32,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; @@ -41,6 +45,7 @@ import org.junit.jupiter.api.Test; import static com.kingsrook.qqq.backend.core.utils.TestUtils.APP_NAME_GREETINGS; import static com.kingsrook.qqq.backend.core.utils.TestUtils.APP_NAME_MISCELLANEOUS; import static com.kingsrook.qqq.backend.core.utils.TestUtils.APP_NAME_PEOPLE; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -304,4 +309,131 @@ class QInstanceEnricherTest extends BaseTest } } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testExposedJoinPaths() + { + ////////////////////////// + // no join path => fail // + ////////////////////////// + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin().withJoinTable("B"))); + qInstance.addTable(newTable("B", "id", "aId")); + assertThatThrownBy(() -> new QInstanceEnricher(qInstance).enrich()) + .rootCause() + .hasMessageContaining("Could not infer a joinPath for table [A], exposedJoin to [B]") + .hasMessageContaining("No join connections between these tables exist in this instance."); + } + + ///////////////////////////////// + // multiple join paths => fail // + ///////////////////////////////// + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin().withJoinTable("B"))); + qInstance.addTable(newTable("B", "id", "aId1", "aId2")); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB1").withJoinOn(new JoinOn("id", "aId1")).withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB2").withJoinOn(new JoinOn("id", "aId2")).withType(JoinType.ONE_TO_ONE)); + assertThatThrownBy(() -> new QInstanceEnricher(qInstance).enrich()) + .rootCause() + .hasMessageContaining("Could not infer a joinPath for table [A], exposedJoin to [B]") + .hasMessageContaining("2 join connections exist") + .hasMessageContaining("\nAB1\n") + .hasMessageContaining("\nAB2."); + + //////////////////////////////////////////// + // but if you specify a path, you're good // + //////////////////////////////////////////// + qInstance.getTable("A").getExposedJoins().get(0).setJoinPath(List.of("AB2")); + new QInstanceEnricher(qInstance).enrich(); + assertEquals("B", qInstance.getTable("A").getExposedJoins().get(0).getLabel()); + } + + ///////////////////////////////// + // multiple join paths => fail // + ///////////////////////////////// + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin().withJoinTable("C"))); + qInstance.addTable(newTable("B", "id", "aId")); + qInstance.addTable(newTable("C", "id", "bId", "aId")); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB").withJoinOn(new JoinOn("id", "aId")).withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withLeftTable("B").withRightTable("C").withName("BC").withJoinOn(new JoinOn("id", "bId")).withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("C").withName("AC").withJoinOn(new JoinOn("id", "aId")).withType(JoinType.ONE_TO_ONE)); + assertThatThrownBy(() -> new QInstanceEnricher(qInstance).enrich()) + .rootCause() + .hasMessageContaining("Could not infer a joinPath for table [A], exposedJoin to [C]") + .hasMessageContaining("2 join connections exist") + .hasMessageContaining("\nAB, BC\n") + .hasMessageContaining("\nAC."); + + //////////////////////////////////////////// + // but if you specify a path, you're good // + //////////////////////////////////////////// + qInstance.getTable("A").getExposedJoins().get(0).setJoinPath(List.of("AB", "BC")); + new QInstanceEnricher(qInstance).enrich(); + assertEquals("C", qInstance.getTable("A").getExposedJoins().get(0).getLabel()); + } + + ////////////////////////////////////////////////////////////////////////////////////// + // even if you specify a bogus path, Enricher doesn't care - see validator to care. // + ////////////////////////////////////////////////////////////////////////////////////// + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin().withJoinTable("B").withJoinPath(List.of("not-a-join")))); + qInstance.addTable(newTable("B", "id", "aId")); + new QInstanceEnricher(qInstance).enrich(); + } + + //////////////////////////////////// + // one join path => great success // + //////////////////////////////////// + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.addTable(newTable("A", "id") + .withExposedJoin(new ExposedJoin().withJoinTable("B")) + .withExposedJoin(new ExposedJoin().withJoinTable("C"))); + qInstance.addTable(newTable("B", "id", "aId")); + qInstance.addTable(newTable("C", "id", "bId")); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB").withJoinOn(new JoinOn("id", "aId")).withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withLeftTable("B").withRightTable("C").withName("BC").withJoinOn(new JoinOn("id", "bId")).withType(JoinType.ONE_TO_ONE)); + + new QInstanceEnricher(qInstance).enrich(); + + ExposedJoin exposedJoinAB = qInstance.getTable("A").getExposedJoins().get(0); + assertEquals("B", exposedJoinAB.getLabel()); + assertEquals(List.of("AB"), exposedJoinAB.getJoinPath()); + + ExposedJoin exposedJoinAC = qInstance.getTable("A").getExposedJoins().get(1); + assertEquals("C", exposedJoinAC.getLabel()); + assertEquals(List.of("AB", "BC"), exposedJoinAC.getJoinPath()); + } + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData newTable(String tableName, String... fieldNames) + { + QTableMetaData tableMetaData = new QTableMetaData() + .withName(tableName) + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField(fieldNames[0]); + + for(String fieldName : fieldNames) + { + tableMetaData.addField(new QFieldMetaData(fieldName, QFieldType.INTEGER)); + } + + return (tableMetaData); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 361343cf..e6e50a68 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -50,6 +50,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; @@ -65,6 +68,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; @@ -1671,6 +1675,83 @@ class QInstanceValidatorTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testExposedJoinPaths() + { + assertValidationFailureReasons(qInstance -> qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin())), + "Table A has an exposedJoin that is missing a joinTable name", + "Table A exposedJoin [missingJoinTableName] is missing a label"); + + assertValidationFailureReasons(qInstance -> qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin().withJoinTable("B"))), + "Table A exposedJoin B is referencing an unrecognized table", + "Table A exposedJoin B is missing a label"); + + assertValidationFailureReasons(qInstance -> + { + qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin().withJoinTable("B").withLabel("B").withJoinPath(List.of("notAJoin")))); + qInstance.addTable(newTable("B", "id", "aId")); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB").withType(JoinType.ONE_TO_ONE).withJoinOn(new JoinOn("id", "aId"))); + }, + "does not match a valid join connection in the instance"); + + assertValidationFailureReasons(qInstance -> + { + qInstance.addTable(newTable("A", "id") + .withExposedJoin(new ExposedJoin().withJoinTable("B").withLabel("foo").withJoinPath(List.of("AB"))) + .withExposedJoin(new ExposedJoin().withJoinTable("C").withLabel("foo").withJoinPath(List.of("AC"))) + ); + qInstance.addTable(newTable("B", "id", "aId")); + qInstance.addTable(newTable("C", "id", "aId")); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB").withType(JoinType.ONE_TO_ONE).withJoinOn(new JoinOn("id", "aId"))); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("C").withName("AC").withType(JoinType.ONE_TO_ONE).withJoinOn(new JoinOn("id", "aId"))); + }, + "more than one join labeled: foo"); + + assertValidationFailureReasons(qInstance -> + { + qInstance.addTable(newTable("A", "id") + .withExposedJoin(new ExposedJoin().withJoinTable("B").withLabel("B1").withJoinPath(List.of("AB"))) + .withExposedJoin(new ExposedJoin().withJoinTable("B").withLabel("B2").withJoinPath(List.of("AB"))) + ); + qInstance.addTable(newTable("B", "id", "aId")); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB").withType(JoinType.ONE_TO_ONE).withJoinOn(new JoinOn("id", "aId"))); + }, + "than one join with the joinPath: [AB]"); + + assertValidationSuccess(qInstance -> + { + qInstance.addTable(newTable("A", "id").withExposedJoin(new ExposedJoin().withJoinTable("B").withLabel("B").withJoinPath(List.of("AB")))); + qInstance.addTable(newTable("B", "id", "aId")); + qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB").withType(JoinType.ONE_TO_ONE).withJoinOn(new JoinOn("id", "aId"))); + }); + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData newTable(String tableName, String... fieldNames) + { + QTableMetaData tableMetaData = new QTableMetaData() + .withName(tableName) + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField(fieldNames[0]); + + for(String fieldName : fieldNames) + { + tableMetaData.addField(new QFieldMetaData(fieldName, QFieldType.INTEGER)); + } + + return (tableMetaData); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoinTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoinTest.java new file mode 100644 index 00000000..def4624d --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoinTest.java @@ -0,0 +1,137 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for ExposedJoin + *******************************************************************************/ +class ExposedJoinTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testIsManyOneToOne() + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addTable(new QTableMetaData().withName("A").withExposedJoin(new ExposedJoin().withJoinTable("B").withJoinPath(List.of("AB")))); + qInstance.addTable(new QTableMetaData().withName("B").withExposedJoin(new ExposedJoin().withJoinTable("A").withJoinPath(List.of("AB")))); + qInstance.addJoin(new QJoinMetaData().withName("AB").withLeftTable("A").withRightTable("B").withType(JoinType.ONE_TO_ONE)); + + assertFalse(qInstance.getTable("A").getExposedJoins().get(0).getIsMany()); + assertFalse(qInstance.getTable("B").getExposedJoins().get(0).getIsMany()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testIsManyOneToMany() + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addTable(new QTableMetaData().withName("A").withExposedJoin(new ExposedJoin().withJoinTable("B").withJoinPath(List.of("AB")))); + qInstance.addTable(new QTableMetaData().withName("B").withExposedJoin(new ExposedJoin().withJoinTable("A").withJoinPath(List.of("AB")))); + qInstance.addJoin(new QJoinMetaData().withName("AB").withLeftTable("A").withRightTable("B").withType(JoinType.ONE_TO_MANY)); + + assertTrue(qInstance.getTable("A").getExposedJoins().get(0).getIsMany()); + assertFalse(qInstance.getTable("B").getExposedJoins().get(0).getIsMany()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testIsManyManyToOne() + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addTable(new QTableMetaData().withName("A").withExposedJoin(new ExposedJoin().withJoinTable("B").withJoinPath(List.of("AB")))); + qInstance.addTable(new QTableMetaData().withName("B").withExposedJoin(new ExposedJoin().withJoinTable("A").withJoinPath(List.of("AB")))); + qInstance.addJoin(new QJoinMetaData().withName("AB").withLeftTable("A").withRightTable("B").withType(JoinType.MANY_TO_ONE)); + + assertFalse(qInstance.getTable("A").getExposedJoins().get(0).getIsMany()); + assertTrue(qInstance.getTable("B").getExposedJoins().get(0).getIsMany()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testIsManyOneToOneThroughChain() + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addTable(new QTableMetaData().withName("A").withExposedJoin(new ExposedJoin().withJoinTable("G").withJoinPath(List.of("AB", "BC", "CD", "DE", "EF", "FG")))); + qInstance.addTable(new QTableMetaData().withName("B").withExposedJoin(new ExposedJoin().withJoinTable("G").withJoinPath(List.of("BC", "CD", "DE", "EF", "FG")))); + qInstance.addJoin(new QJoinMetaData().withName("AB").withLeftTable("A").withRightTable("B").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("BC").withLeftTable("B").withRightTable("C").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("CD").withLeftTable("C").withRightTable("D").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("DE").withLeftTable("D").withRightTable("E").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("EF").withLeftTable("E").withRightTable("F").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("FG").withLeftTable("F").withRightTable("G").withType(JoinType.ONE_TO_ONE)); + + assertFalse(qInstance.getTable("A").getExposedJoins().get(0).getIsMany()); + assertFalse(qInstance.getTable("B").getExposedJoins().get(0).getIsMany()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testIsManyOneToManyThroughChain() + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addTable(new QTableMetaData().withName("A").withExposedJoin(new ExposedJoin().withJoinTable("G").withJoinPath(List.of("AB", "BC", "CD", "DE", "EF", "FG")))); + qInstance.addTable(new QTableMetaData().withName("B").withExposedJoin(new ExposedJoin().withJoinTable("E").withJoinPath(List.of("BC", "CD", "DE")))); + qInstance.addTable(new QTableMetaData().withName("F").withExposedJoin(new ExposedJoin().withJoinTable("C").withJoinPath(List.of("FG", "EF", "DE", "CD")))); + qInstance.addJoin(new QJoinMetaData().withName("AB").withLeftTable("A").withRightTable("B").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("BC").withLeftTable("B").withRightTable("C").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("CD").withLeftTable("C").withRightTable("D").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("DE").withLeftTable("D").withRightTable("E").withType(JoinType.ONE_TO_ONE)); + qInstance.addJoin(new QJoinMetaData().withName("EF").withLeftTable("E").withRightTable("F").withType(JoinType.ONE_TO_MANY)); + qInstance.addJoin(new QJoinMetaData().withName("FG").withLeftTable("F").withRightTable("G").withType(JoinType.ONE_TO_ONE)); + + assertTrue(qInstance.getTable("A").getExposedJoins().get(0).getIsMany()); + assertFalse(qInstance.getTable("B").getExposedJoins().get(0).getIsMany()); + assertFalse(qInstance.getTable("F").getExposedJoins().get(0).getIsMany()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java index d1b13a98..909c9f7c 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java @@ -128,38 +128,31 @@ class EnumerationQueryActionTest extends BaseTest QueryInput queryInput = new QueryInput(); queryInput.setTableName("statesEnum"); - queryInput.setSkip(0); - queryInput.setLimit(null); + queryInput.setFilter(new QQueryFilter().withSkip(0).withLimit(null)); QueryOutput queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of("Missouri", "Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(1); - queryInput.setLimit(null); + queryInput.setFilter(new QQueryFilter().withSkip(1).withLimit(null)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of("Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(2); - queryInput.setLimit(null); + queryInput.setFilter(new QQueryFilter().withSkip(2).withLimit(null)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of(), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(null); - queryInput.setLimit(1); + queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(1)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of("Missouri"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(null); - queryInput.setLimit(2); + queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(2)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of("Missouri", "Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(null); - queryInput.setLimit(3); + queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(3)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of("Missouri", "Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(null); - queryInput.setLimit(0); + queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(0)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of(), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java index 34a25a95..23b8ab93 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java @@ -341,14 +341,13 @@ class MemoryBackendModuleTest extends BaseTest { QueryInput queryInput = new QueryInput(); queryInput.setTableName(table.getName()); - queryInput.setLimit(2); + queryInput.setFilter(new QQueryFilter().withLimit(2)); assertEquals(2, new QueryAction().execute(queryInput).getRecords().size()); - queryInput.setLimit(1); + queryInput.setFilter(new QQueryFilter().withLimit(1)); assertEquals(1, new QueryAction().execute(queryInput).getRecords().size()); - queryInput.setSkip(4); - queryInput.setLimit(3); + queryInput.setFilter(new QQueryFilter().withSkip(4).withLimit(3)); assertEquals(0, new QueryAction().execute(queryInput).getRecords().size()); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index b3953b6a..3c78d9f8 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -92,6 +92,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTracking; @@ -548,6 +549,7 @@ public class TestUtils .withFieldName("storeId")) .withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_LINE_ITEM).withJoinName("orderLineItem")) .withAssociation(new Association().withName("extrinsics").withAssociatedTableName(TABLE_NAME_ORDER_EXTRINSIC).withJoinName("orderOrderExtrinsic")) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_LINE_ITEM)) .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) @@ -582,7 +584,7 @@ public class TestUtils .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("orderId", QFieldType.INTEGER)) .withField(new QFieldMetaData("lineNumber", QFieldType.STRING)) - .withField(new QFieldMetaData("sku", QFieldType.STRING)) + .withField(new QFieldMetaData("sku", QFieldType.STRING).withLabel("SKU")) .withField(new QFieldMetaData("quantity", QFieldType.INTEGER)); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableListTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableListTest.java index 677509fd..6136f814 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableListTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableListTest.java @@ -22,9 +22,11 @@ package com.kingsrook.qqq.backend.core.utils.collections; +import java.util.LinkedList; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -48,4 +50,22 @@ class MutableListTest extends BaseTest list.remove(0); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNullInput() + { + List list = new MutableList<>(null); + list.add(1); + assertEquals(1, list.size()); + + MutableList mutableList = new MutableList<>(null, LinkedList::new); + mutableList.add(1); + assertEquals(1, mutableList.size()); + assertEquals(LinkedList.class, mutableList.getUnderlyingList().getClass()); + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMapTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMapTest.java index cc2d160e..44e85544 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMapTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/MutableMapTest.java @@ -22,9 +22,11 @@ package com.kingsrook.qqq.backend.core.utils.collections; +import java.util.LinkedHashMap; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -48,4 +50,22 @@ class MutableMapTest extends BaseTest map.putAll(Map.of("c", 3, "d", 4)); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNullInput() + { + Map map = new MutableMap<>(null); + map.put(1, "one"); + assertEquals(1, map.size()); + + MutableMap mutableMap = new MutableMap<>(null, LinkedHashMap::new); + mutableMap.put(1, "uno"); + assertEquals(1, mutableMap.size()); + assertEquals(LinkedHashMap.class, mutableMap.getUnderlyingMap().getClass()); + } + } \ No newline at end of file diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index f3cd1107..e32ede54 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -228,10 +228,12 @@ public class BaseAPIActionUtil *******************************************************************************/ public QueryOutput doQuery(QTableMetaData table, QueryInput queryInput) throws QException { - QueryOutput queryOutput = new QueryOutput(queryInput); - Integer originalLimit = queryInput.getLimit(); - Integer limit = originalLimit; - Integer skip = queryInput.getSkip(); + QueryOutput queryOutput = new QueryOutput(queryInput); + QQueryFilter filter = queryInput.getFilter(); + + Integer originalLimit = filter == null ? null : filter.getLimit(); + Integer limit = originalLimit; + Integer skip = filter == null ? null : filter.getSkip(); if(limit == null) { @@ -243,10 +245,9 @@ public class BaseAPIActionUtil { try { - QQueryFilter filter = queryInput.getFilter(); - String paramString = buildQueryStringForGet(filter, limit, skip, table.getFields()); - String url = buildTableUrl(table) + paramString; - HttpGet request = new HttpGet(url); + String paramString = buildQueryStringForGet(filter, limit, skip, table.getFields()); + String url = buildTableUrl(table) + paramString; + HttpGet request = new HttpGet(url); QHttpResponse response = makeRequest(table, request); int count = processGetResponse(table, response, queryOutput); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index fc011387..ec7e7488 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -30,9 +30,12 @@ import java.time.Instant; import java.time.LocalDate; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; @@ -203,7 +206,8 @@ public abstract class AbstractRDBMSAction implements QActionInterface { StringBuilder rs = new StringBuilder(escapeIdentifier(getTableName(instance.getTable(tableName))) + " AS " + escapeIdentifier(tableName)); - for(QueryJoin queryJoin : joinsContext.getQueryJoins()) + List queryJoins = sortQueryJoinsForFromClause(tableName, joinsContext.getQueryJoins()); + for(QueryJoin queryJoin : queryJoins) { QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable()); String tableNameOrAlias = queryJoin.getJoinTableOrItsAlias(); @@ -264,6 +268,48 @@ public abstract class AbstractRDBMSAction implements QActionInterface + /******************************************************************************* + ** We've seen some SQL dialects (mysql, but not h2...) be unhappy if we have + ** the from/join-ons out of order. This method resorts the joins, to start with + ** main table, then any tables attached to it, then fanning out from there. + *******************************************************************************/ + private List sortQueryJoinsForFromClause(String mainTableName, List queryJoins) + { + List inputListCopy = new ArrayList<>(queryJoins); + + List rs = new ArrayList<>(); + Set seenTables = new HashSet<>(); + seenTables.add(mainTableName); + + boolean keepGoing = true; + while(!inputListCopy.isEmpty() && keepGoing) + { + keepGoing = false; + Iterator iterator = inputListCopy.iterator(); + while(iterator.hasNext()) + { + QueryJoin next = iterator.next(); + if((StringUtils.hasContent(next.getBaseTableOrAlias()) && seenTables.contains(next.getBaseTableOrAlias())) || seenTables.contains(next.getJoinTable())) + { + rs.add(next); + if(StringUtils.hasContent(next.getBaseTableOrAlias())) + { + seenTables.add(next.getBaseTableOrAlias()); + } + seenTables.add(next.getJoinTable()); + iterator.remove(); + keepGoing = true; + } + } + } + + rs.addAll(inputListCopy); + + return (rs); + } + + + /******************************************************************************* ** method that sub-classes should call to make a full WHERE clause, including ** security clauses. @@ -902,6 +948,16 @@ public abstract class AbstractRDBMSAction implements QActionInterface + /******************************************************************************* + ** Make it easy (e.g., for tests) to turn on logging of SQL + *******************************************************************************/ + public static void setLogSQLOutput(String loggerOrSystemOut) + { + System.setProperty("qqq.rdbms.logSQL.output", loggerOrSystemOut); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java index dad1067f..e713c1b5 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import org.apache.commons.lang.BooleanUtils; /******************************************************************************* @@ -63,6 +64,11 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf String primaryKeyColumn = escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(fieldAndTableNameOrAlias.field().getName()); String clausePrefix = (requiresDistinct) ? "SELECT COUNT(DISTINCT (" + primaryKeyColumn + "))" : "SELECT COUNT(*)"; + if(BooleanUtils.isTrue(countInput.getIncludeDistinctCount())) + { + clausePrefix = "SELECT COUNT(DISTINCT (" + primaryKeyColumn + ")) AS distinct_count, COUNT(*)"; + } + String sql = clausePrefix + " AS record_count FROM " + makeFromClause(countInput.getInstance(), table.getName(), joinsContext); @@ -81,6 +87,11 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf if(resultSet.next()) { rs.setCount(resultSet.getInt("record_count")); + + if(BooleanUtils.isTrue(countInput.getIncludeDistinctCount())) + { + rs.setDistinctCount(resultSet.getInt("distinct_count")); + } } }), params); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 412dd35a..4522282f 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -83,14 +83,14 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf sql.append(" ORDER BY ").append(makeOrderByClause(table, filter.getOrderBys(), joinsContext)); } - if(queryInput.getLimit() != null) + if(filter != null && filter.getLimit() != null) { - sql.append(" LIMIT ").append(queryInput.getLimit()); + sql.append(" LIMIT ").append(filter.getLimit()); - if(queryInput.getSkip() != null) + if(filter.getSkip() != null) { // todo - other sql grammars? - sql.append(" OFFSET ").append(queryInput.getSkip()); + sql.append(" OFFSET ").append(filter.getSkip()); } } @@ -200,7 +200,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf List queryJoins = queryInput.getQueryJoins(); QTableMetaData table = instance.getTable(tableName); - boolean requiresDistinct = doesSelectClauseRequireDistinct(table); + boolean requiresDistinct = queryInput.getSelectDistinct() || doesSelectClauseRequireDistinct(table); String clausePrefix = (requiresDistinct) ? "SELECT DISTINCT " : "SELECT "; List fieldList = new ArrayList<>(table.getFields().values()); diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java index 0dad47e0..a2479159 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java @@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; @@ -239,6 +240,7 @@ public class TestUtils qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER, "order") .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId")) .withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_ORDER_LINE).withJoinName("orderJoinOrderLine")) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ITEM).withJoinPath(List.of("orderJoinOrderLine", "orderLineJoinItem"))) .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE)) .withField(new QFieldMetaData("billToPersonId", QFieldType.INTEGER).withBackendName("bill_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) .withField(new QFieldMetaData("shipToPersonId", QFieldType.INTEGER).withBackendName("ship_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) @@ -246,7 +248,9 @@ public class TestUtils qInstance.addTable(defineBaseTable(TABLE_NAME_ITEM, "item") .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId")) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER).withJoinPath(List.of("orderLineJoinItem", "orderJoinOrderLine"))) .withField(new QFieldMetaData("sku", QFieldType.STRING)) + .withField(new QFieldMetaData("description", QFieldType.STRING)) .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE)) ); diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java index 4d846e19..637601fa 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java @@ -51,6 +51,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -72,6 +73,20 @@ public class RDBMSQueryActionTest extends RDBMSActionTest public void beforeEach() throws Exception { super.primeTestDatabase(); + + // AbstractRDBMSAction.setLogSQL(true); + // AbstractRDBMSAction.setLogSQLOutput("system.out"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + AbstractRDBMSAction.setLogSQL(false); } @@ -1103,6 +1118,149 @@ public class RDBMSQueryActionTest extends RDBMSActionTest + /******************************************************************************* + ** Given tables: + ** order - orderLine - item + ** with exposedJoin on order to item + ** do a query on order, also selecting item. + *******************************************************************************/ + @Test + void testTwoTableAwayExposedJoin() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + queryInput.withQueryJoins(List.of( + new QueryJoin(TestUtils.TABLE_NAME_ITEM).withType(QueryJoin.Type.INNER).withSelect(true) + )); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + assertThat(records).hasSize(11); // one per line item + assertThat(records).allMatch(r -> r.getValue("id") != null); + assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ITEM + ".description") != null); + } + + + + /******************************************************************************* + ** Given tables: + ** order - orderLine - item + ** with exposedJoin on item to order + ** do a query on item, also selecting order. + ** This is a reverse of the above, to make sure join flipping, etc, is good. + *******************************************************************************/ + @Test + void testTwoTableAwayExposedJoinReversed() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ITEM); + + queryInput.withQueryJoins(List.of( + new QueryJoin(TestUtils.TABLE_NAME_ORDER).withType(QueryJoin.Type.INNER).withSelect(true) + )); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + assertThat(records).hasSize(11); // one per line item + assertThat(records).allMatch(r -> r.getValue("description") != null); + assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ORDER + ".id") != null); + } + + + + /******************************************************************************* + ** Given tables: + ** order - orderLine - item + ** with exposedJoin on order to item + ** do a query on order, also selecting item, and also selecting orderLine... + *******************************************************************************/ + @Test + void testTwoTableAwayExposedJoinAlsoSelectingInBetweenTable() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + queryInput.withQueryJoins(List.of( + new QueryJoin(TestUtils.TABLE_NAME_ORDER_LINE).withType(QueryJoin.Type.INNER).withSelect(true), + new QueryJoin(TestUtils.TABLE_NAME_ITEM).withType(QueryJoin.Type.INNER).withSelect(true) + )); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + assertThat(records).hasSize(11); // one per line item + assertThat(records).allMatch(r -> r.getValue("id") != null); + assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ORDER_LINE + ".quantity") != null); + assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ITEM + ".description") != null); + } + + + + /******************************************************************************* + ** Given tables: + ** order - orderLine - item + ** with exposedJoin on order to item + ** do a query on order, filtered by item + *******************************************************************************/ + @Test + void testTwoTableAwayExposedJoinWhereClauseOnly() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria(TestUtils.TABLE_NAME_ITEM + ".description", QCriteriaOperator.STARTS_WITH, "Q-Mart"))); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + assertThat(records).hasSize(4); + assertThat(records).allMatch(r -> r.getValue("id") != null); + } + + + + /******************************************************************************* + ** Given tables: + ** order - orderLine - item + ** with exposedJoin on order to item + ** do a query on order, filtered by item + *******************************************************************************/ + @Test + void testTwoTableAwayExposedJoinWhereClauseBothJoinTables() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria(TestUtils.TABLE_NAME_ITEM + ".description", QCriteriaOperator.STARTS_WITH, "Q-Mart")) + .withCriteria(new QFilterCriteria(TestUtils.TABLE_NAME_ORDER_LINE + ".quantity", QCriteriaOperator.IS_NOT_BLANK)) + ); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + assertThat(records).hasSize(4); + assertThat(records).allMatch(r -> r.getValue("id") != null); + } + + + /******************************************************************************* ** queries on the store table, where the primary key (id) is the security field *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql b/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql index eb2abfe2..92c69194 100644 --- a/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql +++ b/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql @@ -102,19 +102,20 @@ CREATE TABLE item ( id INT AUTO_INCREMENT PRIMARY KEY, sku VARCHAR(80) NOT NULL, + description VARCHAR(80), store_id INT NOT NULL REFERENCES store ); -- three items for each store -INSERT INTO item (id, sku, store_id) VALUES (1, 'QM-1', 1); -INSERT INTO item (id, sku, store_id) VALUES (2, 'QM-2', 1); -INSERT INTO item (id, sku, store_id) VALUES (3, 'QM-3', 1); -INSERT INTO item (id, sku, store_id) VALUES (4, 'QRU-1', 2); -INSERT INTO item (id, sku, store_id) VALUES (5, 'QRU-2', 2); -INSERT INTO item (id, sku, store_id) VALUES (6, 'QRU-3', 2); -INSERT INTO item (id, sku, store_id) VALUES (7, 'QD-1', 3); -INSERT INTO item (id, sku, store_id) VALUES (8, 'QD-2', 3); -INSERT INTO item (id, sku, store_id) VALUES (9, 'QD-3', 3); +INSERT INTO item (id, sku, description, store_id) VALUES (1, 'QM-1', 'Q-Mart Item 1', 1); +INSERT INTO item (id, sku, description, store_id) VALUES (2, 'QM-2', 'Q-Mart Item 2', 1); +INSERT INTO item (id, sku, description, store_id) VALUES (3, 'QM-3', 'Q-Mart Item 3', 1); +INSERT INTO item (id, sku, description, store_id) VALUES (4, 'QRU-1', 'QQQ R Us Item 4', 2); +INSERT INTO item (id, sku, description, store_id) VALUES (5, 'QRU-2', 'QQQ R Us Item 5', 2); +INSERT INTO item (id, sku, description, store_id) VALUES (6, 'QRU-3', 'QQQ R Us Item 6', 2); +INSERT INTO item (id, sku, description, store_id) VALUES (7, 'QD-1', 'QDepot Item 7', 3); +INSERT INTO item (id, sku, description, store_id) VALUES (8, 'QD-2', 'QDepot Item 8', 3); +INSERT INTO item (id, sku, description, store_id) VALUES (9, 'QD-3', 'QDepot Item 9', 3); CREATE TABLE `order` ( diff --git a/qqq-middleware-javalin/pom.xml b/qqq-middleware-javalin/pom.xml index d225ee0f..78e38af5 100644 --- a/qqq-middleware-javalin/pom.xml +++ b/qqq-middleware-javalin/pom.xml @@ -60,7 +60,7 @@ io.javalin javalin - 5.1.4 + 5.4.2 com.konghq diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 62c88550..ddf3324c 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -788,6 +788,7 @@ public class QJavalinImplementation } countInput.setQueryJoins(processQueryJoinsParam(context)); + countInput.setIncludeDistinctCount(QJavalinUtils.queryParamIsTrue(context, "includeDistinct")); CountAction countAction = new CountAction(); CountOutput countOutput = countAction.execute(countInput); @@ -842,8 +843,6 @@ public class QJavalinImplementation queryInput.setTableName(table); queryInput.setShouldGenerateDisplayValues(true); queryInput.setShouldTranslatePossibleValues(true); - queryInput.setSkip(QJavalinUtils.integerQueryParam(context, "skip")); - queryInput.setLimit(QJavalinUtils.integerQueryParam(context, "limit")); PermissionsHelper.checkTablePermissionThrowing(queryInput, TablePermissionSubType.READ); @@ -857,6 +856,18 @@ public class QJavalinImplementation queryInput.setFilter(JsonUtils.toObject(filter, QQueryFilter.class)); } + Integer skip = QJavalinUtils.integerQueryParam(context, "skip"); + Integer limit = QJavalinUtils.integerQueryParam(context, "limit"); + if(skip != null || limit != null) + { + if(queryInput.getFilter() == null) + { + queryInput.setFilter(new QQueryFilter()); + } + queryInput.getFilter().setSkip(skip); + queryInput.getFilter().setLimit(limit); + } + queryInput.setQueryJoins(processQueryJoinsParam(context)); QueryAction queryAction = new QueryAction(); diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinScriptsHandler.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinScriptsHandler.java index eaaca7ae..57c730ee 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinScriptsHandler.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinScriptsHandler.java @@ -254,8 +254,7 @@ public class QJavalinScriptsHandler queryInput.setFilter(new QQueryFilter() .withCriteria(new QFilterCriteria("scriptRevisionId", QCriteriaOperator.EQUALS, List.of(scriptRevisionId))) .withOrderBy(new QFilterOrderBy("id", false)) - ); - queryInput.setLimit(100); + .withLimit(100)); QueryOutput queryOutput = new QueryAction().execute(queryInput); if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords())) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinUtils.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinUtils.java index d8fa1ebe..fb1a4681 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinUtils.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinUtils.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.javalin; +import java.util.Objects; import com.kingsrook.qqq.backend.core.exceptions.QValueException; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -52,6 +53,23 @@ public class QJavalinUtils + /******************************************************************************* + ** Returns true iff context has a valid query parameter by the given name, with + ** a value of "true". + *******************************************************************************/ + public static boolean queryParamIsTrue(Context context, String name) throws QValueException + { + String value = context.queryParam(name); + if(Objects.equals(value, "true")) + { + return (true); + } + + return (false); + } + + + /******************************************************************************* ** Returns Integer if context has a valid int form parameter by the given name, ** Returns null if no param (or empty value). diff --git a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java index b636d735..093a5f0d 100644 --- a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java +++ b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java @@ -531,7 +531,6 @@ public class QPicoCliImplementation { QueryInput queryInput = new QueryInput(); queryInput.setTableName(tableName); - queryInput.setSkip(subParseResult.matchedOptionValue("skip", null)); // todo - think about these (e.g., based on user's requested output format? // queryInput.setShouldGenerateDisplayValues(true); @@ -553,6 +552,8 @@ public class QPicoCliImplementation .withValues(List.of(primaryKeyValue))); queryInput.setFilter(filter); + filter.setSkip(subParseResult.matchedOptionValue("skip", null)); + QueryAction queryAction = new QueryAction(); QueryOutput queryOutput = queryAction.execute(queryInput); List records = queryOutput.getRecords(); @@ -577,9 +578,9 @@ public class QPicoCliImplementation { QueryInput queryInput = new QueryInput(); queryInput.setTableName(tableName); - queryInput.setSkip(subParseResult.matchedOptionValue("skip", null)); - queryInput.setLimit(subParseResult.matchedOptionValue("limit", null)); queryInput.setFilter(generateQueryFilter(subParseResult)); + queryInput.getFilter().setSkip(subParseResult.matchedOptionValue("skip", null)); + queryInput.getFilter().setLimit(subParseResult.matchedOptionValue("limit", null)); // todo - think about these (e.g., based on user's requested output format? // queryInput.setShouldGenerateDisplayValues(true); diff --git a/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java b/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java index 82527128..56ee591c 100644 --- a/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java +++ b/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java @@ -52,6 +52,7 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; @@ -384,7 +385,7 @@ public class QSlackImplementation try { QueryInput queryInput = new QueryInput(); - queryInput.setLimit(10); + queryInput.setFilter(new QQueryFilter().withLimit(10)); queryInput.setTableName(tableName); setupSession(context, queryInput); QueryOutput output = new QueryAction().execute(queryInput);