From da17145f66d5fa535d11cab0044ef4e266a46074 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 27 Feb 2023 10:30:54 -0600 Subject: [PATCH 1/9] WIP version of table/column stats process & supporting aggregate changes --- .../core/instances/QInstanceEnricher.java | 9 -- .../tables/aggregate/AggregateInput.java | 35 +++++ .../tables/aggregate/AggregateOperator.java | 34 ++++- .../actions/tables/aggregate/GroupBy.java | 11 ++ .../metadata/processes/QProcessMetaData.java | 13 ++ .../tablestats/TableStatsStep.java | 123 ++++++++++++++++++ .../rdbms/actions/RDBMSAggregateAction.java | 7 +- 7 files changed, 217 insertions(+), 15 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 69fc7705..f82740f0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -51,7 +51,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; @@ -293,14 +292,6 @@ public class QInstanceEnricher *******************************************************************************/ private void enrichStep(QStepMetaData step) { - if(!StringUtils.hasContent(step.getName()) && step instanceof QBackendStepMetaData backendStep) - { - if(backendStep.getCode() != null && backendStep.getCode().getName() != null) - { - step.setName(backendStep.getCode().getName()); - } - } - if(!StringUtils.hasContent(step.getLabel())) { step.setLabel(nameToLabel(step.getName())); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateInput.java index 6e292b4e..04459f04 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateInput.java @@ -38,6 +38,7 @@ public class AggregateInput extends AbstractTableActionInput private QQueryFilter filter; private List aggregates; private List groupBys = new ArrayList<>(); + private Integer limit; private List queryJoins = null; @@ -234,4 +235,38 @@ public class AggregateInput extends AbstractTableActionInput return (this); } + + + /******************************************************************************* + ** Getter for limit + ** + *******************************************************************************/ + public Integer getLimit() + { + return limit; + } + + + + /******************************************************************************* + ** Setter for limit + ** + *******************************************************************************/ + public void setLimit(Integer limit) + { + this.limit = limit; + } + + + + /******************************************************************************* + ** Fluent setter for limit + ** + *******************************************************************************/ + public AggregateInput withLimit(Integer limit) + { + this.limit = limit; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateOperator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateOperator.java index a83724ac..513e95c7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateOperator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateOperator.java @@ -27,9 +27,33 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.aggregate; *******************************************************************************/ public enum AggregateOperator { - COUNT, - SUM, - MIN, - MAX, - AVG + COUNT("COUNT("), + COUNT_DISTINCT("COUNT(DISTINCT "), + SUM("SUM("), + MIN("MIN("), + MAX("MAX("), + AVG("AVG("); + + private final String sqlPrefix; + + + + /******************************************************************************* + ** + *******************************************************************************/ + AggregateOperator(String sqlPrefix) + { + this.sqlPrefix = sqlPrefix; + } + + + + /******************************************************************************* + ** Getter for sqlPrefix + ** + *******************************************************************************/ + public String getSqlPrefix() + { + return sqlPrefix; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java index ce28e21c..750fc91b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java @@ -38,6 +38,17 @@ public class GroupBy implements Serializable + /******************************************************************************* + ** + *******************************************************************************/ + public GroupBy(QFieldType type, String fieldName) + { + this.type = type; + this.fieldName = fieldName; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index dd0ad81c..964720cb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -234,6 +235,12 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi { this.steps = new HashMap<>(); } + + if(!StringUtils.hasContent(step.getName())) + { + throw (new IllegalArgumentException("Attempt to add a process step without a name")); + } + this.steps.put(step.getName(), step); return (this); @@ -251,6 +258,12 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi { this.steps = new HashMap<>(); } + + if(!StringUtils.hasContent(step.getName())) + { + throw (new IllegalArgumentException("Attempt to add a process step without a name")); + } + this.steps.put(step.getName(), step); return (this); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java new file mode 100644 index 00000000..9687bcb9 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java @@ -0,0 +1,123 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.tablestats; + + +import java.io.Serializable; +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TableStatsStep implements BackendStep +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + String tableName = runBackendStepInput.getValueString("tableName"); + String fieldName = runBackendStepInput.getValueString("fieldName"); + String filterJSON = runBackendStepInput.getValueString("filterJSON"); + + QQueryFilter filter = null; + if(StringUtils.hasContent(filterJSON)) + { + filter = JsonUtils.toObject(filterJSON, QQueryFilter.class); + } + else + { + filter = new QQueryFilter(); + } + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + QFieldMetaData field = table.getField(fieldName); + + Aggregate aggregate = new Aggregate(fieldName, AggregateOperator.COUNT); + GroupBy groupBy = new GroupBy(field.getType(), fieldName); + + Integer limit = 1000; // too big? + AggregateInput aggregateInput = new AggregateInput(); + aggregateInput.withAggregate(aggregate); + aggregateInput.withGroupBy(groupBy); + aggregateInput.setTableName(tableName); + aggregateInput.setFilter(filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false))); + aggregateInput.setLimit(limit); + AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput); + + ArrayList valueCounts = new ArrayList<>(); + for(AggregateResult result : aggregateOutput.getResults()) + { + Serializable value = result.getGroupByValue(groupBy); + Integer count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate)); + valueCounts.add(new QRecord().withValue("value", value).withValue("count", count)); + } + runBackendStepOutput.addValue("valueCounts", valueCounts); + + if(valueCounts.size() < limit) + { + runBackendStepOutput.addValue("countDistinct", valueCounts.size()); + } + else + { + Aggregate countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT); + AggregateInput countDistinctAggregateInput = new AggregateInput(); + countDistinctAggregateInput.withAggregate(countDistinctAggregate); + countDistinctAggregateInput.setTableName(tableName); + countDistinctAggregateInput.setFilter(filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false))); + AggregateOutput countDistinctAggregateOutput = new AggregateAction().execute(countDistinctAggregateInput); + AggregateResult countDistinctAggregateResult = countDistinctAggregateOutput.getResults().get(0); + runBackendStepOutput.addValue("countDistinct", countDistinctAggregateResult.getAggregateValue(countDistinctAggregate)); + } + } + catch(Exception e) + { + throw new QException("Error calculating stats", e); + } + } + +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java index f5eaf881..fec2d3a1 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java @@ -85,6 +85,11 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega sql += " ORDER BY " + makeOrderByClause(table, filter.getOrderBys(), joinsContext); } + if(aggregateInput.getLimit() != null) + { + sql += " LIMIT " + aggregateInput.getLimit(); + } + // todo sql customization - can edit sql and/or param list AggregateOutput rs = new AggregateOutput(); @@ -155,7 +160,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega for(Aggregate aggregate : aggregateInput.getAggregates()) { JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(aggregate.getFieldName()); - rs.add(aggregate.getOperator() + "(" + escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(fieldAndTableNameOrAlias.field())) + ")"); + rs.add(aggregate.getOperator().getSqlPrefix() + escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(fieldAndTableNameOrAlias.field())) + ")"); } return (rs); } From dd28c95fc003f4f6036b64fc14e159c3644c6783 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 1 Mar 2023 08:36:03 -0600 Subject: [PATCH 2/9] Use sessino from context, not input --- .../permissions/PermissionsHelper.java | 6 ++-- .../values/QPossibleValueTranslator.java | 34 +++++++++++-------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java index 3f79d968..f076f682 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java @@ -436,7 +436,7 @@ public class PermissionsHelper } } - if(hasPermission(actionInput.getSession(), permissionBaseName, effectivePermissionSubType)) + if(hasPermission(QContext.getQSession(), permissionBaseName, effectivePermissionSubType)) { return (PermissionCheckResult.ALLOW); } @@ -534,9 +534,9 @@ public class PermissionsHelper return; } - if(!hasPermission(actionInput.getSession(), permissionBaseName, effectivePermissionSubType)) + if(!hasPermission(QContext.getQSession(), permissionBaseName, effectivePermissionSubType)) { - // LOG.debug("Throwing permission denied for: " + getPermissionName(permissionBaseName, effectivePermissionSubType) + " for " + actionInput.getSession().getUser()); + // LOG.debug("Throwing permission denied for: " + getPermissionName(permissionBaseName, effectivePermissionSubType) + " for " + QContext.getQSession().getUser()); throw (new QPermissionDeniedException("Permission denied.")); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java index 4da2ff62..9d7ed1a7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java @@ -34,6 +34,7 @@ import java.util.Objects; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QValueException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; @@ -67,28 +68,31 @@ public class QPossibleValueTranslator { private static final QLogger LOG = QLogger.getLogger(QPossibleValueTranslator.class); - private final QInstance qInstance; - private final QSession session; - /////////////////////////////////////////////////////// // top-level keys are pvsNames (not table names) // // 2nd-level keys are pkey values from the PVS table // /////////////////////////////////////////////////////// - private Map> possibleValueCache; + private Map> possibleValueCache = new HashMap<>(); // todo not commit - remove instance & session - use Context + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public QPossibleValueTranslator() + { + } + + + /******************************************************************************* ** *******************************************************************************/ public QPossibleValueTranslator(QInstance qInstance, QSession session) { - this.qInstance = qInstance; - this.session = session; - - this.possibleValueCache = new HashMap<>(); } @@ -141,7 +145,7 @@ public class QPossibleValueTranslator { try { - QTableMetaData joinTable = qInstance.getTable(queryJoin.getJoinTable()); + QTableMetaData joinTable = QContext.getQInstance().getTable(queryJoin.getJoinTable()); for(QFieldMetaData field : joinTable.getFields().values()) { String joinFieldName = Objects.requireNonNullElse(queryJoin.getAlias(), joinTable.getName()) + "." + field.getName(); @@ -152,7 +156,7 @@ public class QPossibleValueTranslator /////////////////////////////////////////////// // avoid circling-back upon the source table // /////////////////////////////////////////////// - QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(field.getPossibleValueSourceName()); + QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName()); if(QPossibleValueSourceType.TABLE.equals(possibleValueSource.getType()) && table.getName().equals(possibleValueSource.getTableName())) { continue; @@ -212,7 +216,7 @@ public class QPossibleValueTranslator *******************************************************************************/ public String translatePossibleValue(QFieldMetaData field, Serializable value) { - QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(field.getPossibleValueSourceName()); + QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName()); if(possibleValueSource == null) { LOG.error("Missing possible value source named [" + field.getPossibleValueSourceName() + "] when formatting value for field [" + field.getName() + "]"); @@ -414,7 +418,7 @@ public class QPossibleValueTranslator if(queryJoin.getSelect()) { String aliasOrTableName = Objects.requireNonNullElse(queryJoin.getAlias(), queryJoin.getJoinTable()); - primePvsCacheTableListingHashLoader(qInstance.getTable(queryJoin.getJoinTable()), fieldsByPvsTable, pvsesByTable, aliasOrTableName + ".", queryJoin.getJoinTable(), limitedToFieldNames); + primePvsCacheTableListingHashLoader(QContext.getQInstance().getTable(queryJoin.getJoinTable()), fieldsByPvsTable, pvsesByTable, aliasOrTableName + ".", queryJoin.getJoinTable(), limitedToFieldNames); } } @@ -474,7 +478,7 @@ public class QPossibleValueTranslator { for(QFieldMetaData field : table.getFields().values()) { - QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(field.getPossibleValueSourceName()); + QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName()); if(possibleValueSource != null && possibleValueSource.getType().equals(QPossibleValueSourceType.TABLE)) { if(limitedToFieldNames != null && !limitedToFieldNames.contains(fieldNamePrefix + field.getName())) @@ -510,7 +514,7 @@ public class QPossibleValueTranslator try { - String primaryKeyField = qInstance.getTable(tableName).getPrimaryKeyField(); + String primaryKeyField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField(); for(List page : CollectionUtils.getPages(values, 1000)) { @@ -531,7 +535,7 @@ public class QPossibleValueTranslator { if(possibleValueSource.getType().equals(QPossibleValueSourceType.TABLE)) { - QTableMetaData table = qInstance.getTable(possibleValueSource.getTableName()); + QTableMetaData table = QContext.getQInstance().getTable(possibleValueSource.getTableName()); for(String recordLabelField : CollectionUtils.nonNullList(table.getRecordLabelFields())) { QFieldMetaData field = table.getField(recordLabelField); From 001ec3a34a317a9364134589d4a2e70c00c12fd5 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 1 Mar 2023 08:36:53 -0600 Subject: [PATCH 3/9] Add overload that works on list of fields rather than table --- .../core/actions/values/QValueFormatter.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java index ddfa92aa..f47029dc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -282,7 +282,8 @@ public class QValueFormatter /******************************************************************************* - ** For a list of records, set their recordLabels and display values + ** For a list of records, set their recordLabels and display values - including + ** record label (e.g., from the table meta data). *******************************************************************************/ public static void setDisplayValuesInRecords(QTableMetaData table, List records) { @@ -300,6 +301,24 @@ public class QValueFormatter + /******************************************************************************* + ** For a list of records, set their recordLabels and display values + *******************************************************************************/ + public static void setDisplayValuesInRecords(Collection fields, List records) + { + if(records == null) + { + return; + } + + for(QRecord record : records) + { + setDisplayValuesInRecord(fields, record); + } + } + + + /******************************************************************************* ** For a list of records, set their display values *******************************************************************************/ From bbde64b02d5c406e007a0c4e0906da2914577bcb Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 1 Mar 2023 08:37:11 -0600 Subject: [PATCH 4/9] Add table permission check; add display & possible values; --- .../tablestats/TableStatsStep.java | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java index 9687bcb9..84f6f9fc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java @@ -24,8 +24,13 @@ package com.kingsrook.qqq.backend.core.processes.implementations.tablestats; import java.io.Serializable; import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; +import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; @@ -38,8 +43,11 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateRe import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -64,10 +72,20 @@ public class TableStatsStep implements BackendStep String fieldName = runBackendStepInput.getValueString("fieldName"); String filterJSON = runBackendStepInput.getValueString("filterJSON"); + ///////////////////////////////////////// + // make sure user may query this table // + ///////////////////////////////////////// + PermissionsHelper.checkTablePermissionThrowing(new QueryInput().withTableName(tableName), TablePermissionSubType.READ); + QQueryFilter filter = null; if(StringUtils.hasContent(filterJSON)) { filter = JsonUtils.toObject(filterJSON, QQueryFilter.class); + + /////////////////////////////////////////////////////////////// + // ... remove any order-by that may have been in that filter // + /////////////////////////////////////////////////////////////// + filter.setOrderBys(new ArrayList<>()); } else { @@ -94,8 +112,14 @@ public class TableStatsStep implements BackendStep { Serializable value = result.getGroupByValue(groupBy); Integer count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate)); - valueCounts.add(new QRecord().withValue("value", value).withValue("count", count)); + valueCounts.add(new QRecord().withValue(fieldName, value).withValue("count", count)); } + QFieldMetaData countField = new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS).withLabel("Count"); + + QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(); + qPossibleValueTranslator.translatePossibleValuesInRecords(table, valueCounts, null, null); + QValueFormatter.setDisplayValuesInRecords(List.of(table.getField(fieldName), countField), valueCounts); + runBackendStepOutput.addValue("valueCounts", valueCounts); if(valueCounts.size() < limit) From ad2eff0e73c7e77d402d54739808a4740077226d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 3 Mar 2023 09:51:22 -0600 Subject: [PATCH 5/9] More stats & aggregates --- .../tablestats/TableStatsStep.java | 185 ++++++++++++++++-- .../rdbms/actions/RDBMSAggregateAction.java | 7 +- 2 files changed, 179 insertions(+), 13 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java index 84f6f9fc..433e19ac 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java @@ -33,6 +33,8 @@ import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; @@ -42,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOu import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult; import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -49,6 +52,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -59,6 +63,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public class TableStatsStep implements BackendStep { + private static final QLogger LOG = QLogger.getLogger(TableStatsStep.class); + + /******************************************************************************* ** @@ -70,6 +77,7 @@ public class TableStatsStep implements BackendStep { String tableName = runBackendStepInput.getValueString("tableName"); String fieldName = runBackendStepInput.getValueString("fieldName"); + String orderBy = runBackendStepInput.getValueString("orderBy"); String filterJSON = runBackendStepInput.getValueString("filterJSON"); ///////////////////////////////////////// @@ -95,15 +103,48 @@ public class TableStatsStep implements BackendStep QTableMetaData table = QContext.getQInstance().getTable(tableName); QFieldMetaData field = table.getField(fieldName); - Aggregate aggregate = new Aggregate(fieldName, AggregateOperator.COUNT); + //////////////////////////////////////////// + // do a count query grouped by this field // + //////////////////////////////////////////// + Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT); GroupBy groupBy = new GroupBy(field.getType(), fieldName); + if(StringUtils.hasContent(orderBy)) + { + if(orderBy.equalsIgnoreCase("count.asc")) + { + filter.withOrderBy(new QFilterOrderByAggregate(aggregate, true)); + } + else if(orderBy.equalsIgnoreCase("count.desc")) + { + filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)); + } + else if(orderBy.equalsIgnoreCase(fieldName + ".asc")) + { + filter.withOrderBy(new QFilterOrderBy(fieldName, true)); + } + else if(orderBy.equalsIgnoreCase(fieldName + ".desc")) + { + filter.withOrderBy(new QFilterOrderBy(fieldName, false)); + } + else + { + LOG.info("Unrecognized orderBy: " + orderBy); + } + } + + /////////////////////////////////////////////////////////////////////////////////////////////////// + // always add order by to break ties. these will be the default too, if input didn't supply one // + /////////////////////////////////////////////////////////////////////////////////////////////////// + filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)); + filter.withOrderBy(new QFilterOrderBy(fieldName)); + Integer limit = 1000; // too big? AggregateInput aggregateInput = new AggregateInput(); aggregateInput.withAggregate(aggregate); aggregateInput.withGroupBy(groupBy); aggregateInput.setTableName(tableName); - aggregateInput.setFilter(filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false))); + aggregateInput.setFilter(filter); aggregateInput.setLimit(limit); AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput); @@ -122,21 +163,141 @@ public class TableStatsStep implements BackendStep runBackendStepOutput.addValue("valueCounts", valueCounts); + ///////////////////////////////////////////////////// + // now do individual statistics as a pseudo-record // + ///////////////////////////////////////////////////// + QFieldMetaData countNonNullField = new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS); + QFieldMetaData countDistinctField = new QFieldMetaData("countDistinct", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS); + QFieldMetaData sumField = new QFieldMetaData("sum", QFieldType.DECIMAL).withDisplayFormat(field.getDisplayFormat()); + QFieldMetaData avgField = new QFieldMetaData("average", QFieldType.DECIMAL).withDisplayFormat(field.getDisplayFormat()); + QFieldMetaData minField = new QFieldMetaData("min", field.getType()).withDisplayFormat(field.getDisplayFormat()); + QFieldMetaData maxField = new QFieldMetaData("max", field.getType()).withDisplayFormat(field.getDisplayFormat()); + + boolean doCountDistinct = true; + boolean doSum = true; + boolean doAvg = true; + boolean doMin = true; + boolean doMax = true; + if(field.getType().isStringLike()) + { + doSum = false; + doAvg = false; + } + if(field.getType().equals(QFieldType.BOOLEAN)) + { + doSum = false; + doAvg = false; + doMin = false; + doMax = false; + } + if(field.getType().equals(QFieldType.DATE) || field.getType().equals(QFieldType.DATE_TIME)) + { + doSum = false; + doAvg = false; // could this be done? + } + if(StringUtils.hasContent(field.getPossibleValueSourceName())) + { + doSum = false; + doAvg = false; + doMin = false; + doMax = false; + } + + ArrayList fields = new ArrayList<>(); + fields.add(countNonNullField); + fields.add(countDistinctField); + if(doSum) + { + fields.add(sumField); + } + if(doAvg) + { + fields.add(avgField); + } + if(doMin) + { + fields.add(minField); + } + if(doMax) + { + fields.add(maxField); + } + + QRecord statsRecord = new QRecord(); + if(valueCounts.size() < limit) { - runBackendStepOutput.addValue("countDistinct", valueCounts.size()); + statsRecord.setValue(countDistinctField.getName(), valueCounts.size()); + doCountDistinct = false; } - else + + Aggregate countNonNullAggregate = new Aggregate(fieldName, AggregateOperator.COUNT); + Aggregate countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT); + Aggregate sumAggregate = new Aggregate(fieldName, AggregateOperator.SUM); + Aggregate avgAggregate = new Aggregate(fieldName, AggregateOperator.AVG); + Aggregate minAggregate = new Aggregate(fieldName, AggregateOperator.MIN); + Aggregate maxAggregate = new Aggregate(fieldName, AggregateOperator.MAX); + AggregateInput statsAggregateInput = new AggregateInput(); + statsAggregateInput.withAggregate(countNonNullAggregate); + if(doCountDistinct) { - Aggregate countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT); - AggregateInput countDistinctAggregateInput = new AggregateInput(); - countDistinctAggregateInput.withAggregate(countDistinctAggregate); - countDistinctAggregateInput.setTableName(tableName); - countDistinctAggregateInput.setFilter(filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false))); - AggregateOutput countDistinctAggregateOutput = new AggregateAction().execute(countDistinctAggregateInput); - AggregateResult countDistinctAggregateResult = countDistinctAggregateOutput.getResults().get(0); - runBackendStepOutput.addValue("countDistinct", countDistinctAggregateResult.getAggregateValue(countDistinctAggregate)); + statsAggregateInput.withAggregate(countDistinctAggregate); } + if(doSum) + { + statsAggregateInput.withAggregate(sumAggregate); + } + if(doAvg) + { + statsAggregateInput.withAggregate(avgAggregate); + } + if(doMin) + { + statsAggregateInput.withAggregate(minAggregate); + } + if(doMax) + { + statsAggregateInput.withAggregate(maxAggregate); + } + + if(CollectionUtils.nullSafeHasContents(statsAggregateInput.getAggregates())) + { + statsAggregateInput.setTableName(tableName); + filter.setOrderBys(new ArrayList<>()); + statsAggregateInput.setFilter(filter); + AggregateOutput statsAggregateOutput = new AggregateAction().execute(statsAggregateInput); + AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0); + + statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate)); + if(doCountDistinct) + { + statsRecord.setValue(countDistinctField.getName(), statsAggregateResult.getAggregateValue(countDistinctAggregate)); + } + if(doSum) + { + statsRecord.setValue(sumField.getName(), statsAggregateResult.getAggregateValue(sumAggregate)); + } + if(doAvg) + { + statsRecord.setValue(avgField.getName(), statsAggregateResult.getAggregateValue(avgAggregate)); + } + if(doMin) + { + statsRecord.setValue(minField.getName(), statsAggregateResult.getAggregateValue(minAggregate)); + } + if(doMax) + { + statsRecord.setValue(maxField.getName(), statsAggregateResult.getAggregateValue(maxAggregate)); + } + } + + QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(null); + fields.forEach(qInstanceEnricher::enrichField); + + QValueFormatter.setDisplayValuesInRecord(fields, statsRecord); + + runBackendStepOutput.addValue("statsFields", fields); + runBackendStepOutput.addValue("statsRecord", statsRecord); } catch(Exception e) { diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java index fec2d3a1..ab4a1222 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java @@ -119,7 +119,12 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(aggregate.getFieldName()); QFieldMetaData field = fieldAndTableNameOrAlias.field(); - if(field.getType().equals(QFieldType.INTEGER) && aggregate.getOperator().equals(AggregateOperator.AVG)) + if(field.getType().equals(QFieldType.INTEGER) && (aggregate.getOperator().equals(AggregateOperator.AVG) || aggregate.getOperator().equals(AggregateOperator.SUM))) + { + field = new QFieldMetaData().withType(QFieldType.DECIMAL); + } + + if(aggregate.getOperator().equals(AggregateOperator.COUNT) || aggregate.getOperator().equals(AggregateOperator.COUNT_DISTINCT)) { field = new QFieldMetaData().withType(QFieldType.DECIMAL); } From 93912844792d1afb12340278e7bbd2913c61046e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 3 Mar 2023 10:14:01 -0600 Subject: [PATCH 6/9] Adjust field labels --- .../processes/implementations/tablestats/TableStatsStep.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java index 433e19ac..aeed396b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java @@ -166,8 +166,8 @@ public class TableStatsStep implements BackendStep ///////////////////////////////////////////////////// // now do individual statistics as a pseudo-record // ///////////////////////////////////////////////////// - QFieldMetaData countNonNullField = new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS); - QFieldMetaData countDistinctField = new QFieldMetaData("countDistinct", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS); + QFieldMetaData countNonNullField = new QFieldMetaData("count", QFieldType.INTEGER).withLabel("Rows with a value").withDisplayFormat(DisplayFormat.COMMAS); + QFieldMetaData countDistinctField = new QFieldMetaData("countDistinct", QFieldType.INTEGER).withLabel("Distinct values").withDisplayFormat(DisplayFormat.COMMAS); QFieldMetaData sumField = new QFieldMetaData("sum", QFieldType.DECIMAL).withDisplayFormat(field.getDisplayFormat()); QFieldMetaData avgField = new QFieldMetaData("average", QFieldType.DECIMAL).withDisplayFormat(field.getDisplayFormat()); QFieldMetaData minField = new QFieldMetaData("min", field.getType()).withDisplayFormat(field.getDisplayFormat()); From bf44f9763085469e35e8382a2630cbd9db070fd8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 13 Mar 2023 08:28:46 -0500 Subject: [PATCH 7/9] Renamed tableStats to columnStats --- .../ColumnStatsStep.java} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/{tablestats/TableStatsStep.java => columnstats/ColumnStatsStep.java} (98%) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java similarity index 98% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java index aeed396b..ae6dc962 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.processes.implementations.tablestats; +package com.kingsrook.qqq.backend.core.processes.implementations.columnstats; import java.io.Serializable; @@ -61,9 +61,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; /******************************************************************************* ** *******************************************************************************/ -public class TableStatsStep implements BackendStep +public class ColumnStatsStep implements BackendStep { - private static final QLogger LOG = QLogger.getLogger(TableStatsStep.class); + private static final QLogger LOG = QLogger.getLogger(ColumnStatsStep.class); From 939dcc308c93b8c788695dbe578bb071f1501d36 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 16 Mar 2023 09:06:09 -0500 Subject: [PATCH 8/9] Fix test; add comment --- .../implementations/columnstats/ColumnStatsStep.java | 4 +++- .../backend/core/instances/QInstanceValidatorTest.java | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java index ae6dc962..40777447 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java @@ -59,7 +59,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; /******************************************************************************* - ** + ** This is a single-step process used to provide Column Statistics. These include + ** counts per-value for a field, plus things like total count, min, max, avg, based + ** on the field type. *******************************************************************************/ public class ColumnStatsStep implements BackendStep { diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 61f803bf..0bedf9ae 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -347,11 +347,11 @@ class QInstanceValidatorTest extends BaseTest @Test public void test_validateProcessStepWithEmptyName() { - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // these used to be an assertion failure - but enricher now sets a default name for backend steps w/ a code name // - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - assertValidationSuccess((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).getStepList().get(0).setName("")); - assertValidationSuccess((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().get(1).setName(null)); + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).getStepList().get(0).setName(""), + "Missing name for a step"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().get(1).setName(null), + "Missing name for a step"); } From b16eaca39434c7523f1b1299beafa057c91ac3b4 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 16 Mar 2023 11:38:26 -0500 Subject: [PATCH 9/9] Let caller specify type to use for an aggregate expression --- .../actions/tables/aggregate/Aggregate.java | 42 +++++++++++++++++-- .../columnstats/ColumnStatsStep.java | 20 ++++++--- .../rdbms/actions/RDBMSAggregateAction.java | 12 ++++-- 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/Aggregate.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/Aggregate.java index 5598f154..c6001695 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/Aggregate.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/Aggregate.java @@ -24,15 +24,18 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.aggregate; import java.io.Serializable; import java.util.Objects; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; /******************************************************************************* - ** + ** Define an "aggregate", e.g., to be selected in an Aggregate action. + ** Such as SUM(cost). *******************************************************************************/ public class Aggregate implements Serializable { private String fieldName; private AggregateOperator operator; + private QFieldType fieldType; @@ -55,12 +58,14 @@ public class Aggregate implements Serializable { return true; } + if(o == null || getClass() != o.getClass()) { return false; } + Aggregate aggregate = (Aggregate) o; - return Objects.equals(fieldName, aggregate.fieldName) && operator == aggregate.operator; + return Objects.equals(fieldName, aggregate.fieldName) && operator == aggregate.operator && fieldType == aggregate.fieldType; } @@ -71,7 +76,7 @@ public class Aggregate implements Serializable @Override public int hashCode() { - return Objects.hash(fieldName, operator); + return Objects.hash(fieldName, operator, fieldType); } @@ -153,4 +158,35 @@ public class Aggregate implements Serializable return (this); } + + + /******************************************************************************* + ** Getter for fieldType + *******************************************************************************/ + public QFieldType getFieldType() + { + return (this.fieldType); + } + + + + /******************************************************************************* + ** Setter for fieldType + *******************************************************************************/ + public void setFieldType(QFieldType fieldType) + { + this.fieldType = fieldType; + } + + + + /******************************************************************************* + ** Fluent setter for fieldType + *******************************************************************************/ + public Aggregate withFieldType(QFieldType fieldType) + { + this.fieldType = fieldType; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java index 40777447..24efecdb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java @@ -108,7 +108,7 @@ public class ColumnStatsStep implements BackendStep //////////////////////////////////////////// // do a count query grouped by this field // //////////////////////////////////////////// - Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT); + Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL); GroupBy groupBy = new GroupBy(field.getType(), fieldName); if(StringUtils.hasContent(orderBy)) @@ -171,7 +171,7 @@ public class ColumnStatsStep implements BackendStep QFieldMetaData countNonNullField = new QFieldMetaData("count", QFieldType.INTEGER).withLabel("Rows with a value").withDisplayFormat(DisplayFormat.COMMAS); QFieldMetaData countDistinctField = new QFieldMetaData("countDistinct", QFieldType.INTEGER).withLabel("Distinct values").withDisplayFormat(DisplayFormat.COMMAS); QFieldMetaData sumField = new QFieldMetaData("sum", QFieldType.DECIMAL).withDisplayFormat(field.getDisplayFormat()); - QFieldMetaData avgField = new QFieldMetaData("average", QFieldType.DECIMAL).withDisplayFormat(field.getDisplayFormat()); + QFieldMetaData avgField = new QFieldMetaData("average", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.DECIMAL2_COMMAS); QFieldMetaData minField = new QFieldMetaData("min", field.getType()).withDisplayFormat(field.getDisplayFormat()); QFieldMetaData maxField = new QFieldMetaData("max", field.getType()).withDisplayFormat(field.getDisplayFormat()); @@ -205,6 +205,11 @@ public class ColumnStatsStep implements BackendStep doMax = false; } + if(field.getName().equals(table.getPrimaryKeyField())) + { + doSum = false; + } + ArrayList fields = new ArrayList<>(); fields.add(countNonNullField); fields.add(countDistinctField); @@ -233,10 +238,13 @@ public class ColumnStatsStep implements BackendStep doCountDistinct = false; } - Aggregate countNonNullAggregate = new Aggregate(fieldName, AggregateOperator.COUNT); - Aggregate countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT); - Aggregate sumAggregate = new Aggregate(fieldName, AggregateOperator.SUM); - Aggregate avgAggregate = new Aggregate(fieldName, AggregateOperator.AVG); + ///////////////////////////////////////////////////////////////////////////////// + // just in case any of these don't fit in an integer, use decimal for them all // + ///////////////////////////////////////////////////////////////////////////////// + Aggregate countNonNullAggregate = new Aggregate(fieldName, AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL); + Aggregate countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT).withFieldType(QFieldType.DECIMAL); + Aggregate sumAggregate = new Aggregate(fieldName, AggregateOperator.SUM).withFieldType(QFieldType.DECIMAL); + Aggregate avgAggregate = new Aggregate(fieldName, AggregateOperator.AVG).withFieldType(QFieldType.DECIMAL); Aggregate minAggregate = new Aggregate(fieldName, AggregateOperator.MIN); Aggregate maxAggregate = new Aggregate(fieldName, AggregateOperator.MAX); AggregateInput statsAggregateInput = new AggregateInput(); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java index ab4a1222..b7ba77ea 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java @@ -119,14 +119,18 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(aggregate.getFieldName()); QFieldMetaData field = fieldAndTableNameOrAlias.field(); - if(field.getType().equals(QFieldType.INTEGER) && (aggregate.getOperator().equals(AggregateOperator.AVG) || aggregate.getOperator().equals(AggregateOperator.SUM))) + QFieldType fieldType = aggregate.getFieldType(); + if(fieldType == null) { - field = new QFieldMetaData().withType(QFieldType.DECIMAL); + if(field.getType().equals(QFieldType.INTEGER) && (aggregate.getOperator().equals(AggregateOperator.AVG))) + { + fieldType = QFieldType.DECIMAL; + } } - if(aggregate.getOperator().equals(AggregateOperator.COUNT) || aggregate.getOperator().equals(AggregateOperator.COUNT_DISTINCT)) + if(fieldType != null) { - field = new QFieldMetaData().withType(QFieldType.DECIMAL); + field = new QFieldMetaData().withType(fieldType); } Serializable value = getFieldValueFromResultSet(field, resultSet, selectionIndex++);