diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java index 3f79d968..f076f682 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java @@ -436,7 +436,7 @@ public class PermissionsHelper } } - if(hasPermission(actionInput.getSession(), permissionBaseName, effectivePermissionSubType)) + if(hasPermission(QContext.getQSession(), permissionBaseName, effectivePermissionSubType)) { return (PermissionCheckResult.ALLOW); } @@ -534,9 +534,9 @@ public class PermissionsHelper return; } - if(!hasPermission(actionInput.getSession(), permissionBaseName, effectivePermissionSubType)) + if(!hasPermission(QContext.getQSession(), permissionBaseName, effectivePermissionSubType)) { - // LOG.debug("Throwing permission denied for: " + getPermissionName(permissionBaseName, effectivePermissionSubType) + " for " + actionInput.getSession().getUser()); + // LOG.debug("Throwing permission denied for: " + getPermissionName(permissionBaseName, effectivePermissionSubType) + " for " + QContext.getQSession().getUser()); throw (new QPermissionDeniedException("Permission denied.")); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java index 4da2ff62..9d7ed1a7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java @@ -34,6 +34,7 @@ import java.util.Objects; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QValueException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; @@ -67,28 +68,31 @@ public class QPossibleValueTranslator { private static final QLogger LOG = QLogger.getLogger(QPossibleValueTranslator.class); - private final QInstance qInstance; - private final QSession session; - /////////////////////////////////////////////////////// // top-level keys are pvsNames (not table names) // // 2nd-level keys are pkey values from the PVS table // /////////////////////////////////////////////////////// - private Map> possibleValueCache; + private Map> possibleValueCache = new HashMap<>(); // todo not commit - remove instance & session - use Context + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public QPossibleValueTranslator() + { + } + + + /******************************************************************************* ** *******************************************************************************/ public QPossibleValueTranslator(QInstance qInstance, QSession session) { - this.qInstance = qInstance; - this.session = session; - - this.possibleValueCache = new HashMap<>(); } @@ -141,7 +145,7 @@ public class QPossibleValueTranslator { try { - QTableMetaData joinTable = qInstance.getTable(queryJoin.getJoinTable()); + QTableMetaData joinTable = QContext.getQInstance().getTable(queryJoin.getJoinTable()); for(QFieldMetaData field : joinTable.getFields().values()) { String joinFieldName = Objects.requireNonNullElse(queryJoin.getAlias(), joinTable.getName()) + "." + field.getName(); @@ -152,7 +156,7 @@ public class QPossibleValueTranslator /////////////////////////////////////////////// // avoid circling-back upon the source table // /////////////////////////////////////////////// - QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(field.getPossibleValueSourceName()); + QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName()); if(QPossibleValueSourceType.TABLE.equals(possibleValueSource.getType()) && table.getName().equals(possibleValueSource.getTableName())) { continue; @@ -212,7 +216,7 @@ public class QPossibleValueTranslator *******************************************************************************/ public String translatePossibleValue(QFieldMetaData field, Serializable value) { - QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(field.getPossibleValueSourceName()); + QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName()); if(possibleValueSource == null) { LOG.error("Missing possible value source named [" + field.getPossibleValueSourceName() + "] when formatting value for field [" + field.getName() + "]"); @@ -414,7 +418,7 @@ public class QPossibleValueTranslator if(queryJoin.getSelect()) { String aliasOrTableName = Objects.requireNonNullElse(queryJoin.getAlias(), queryJoin.getJoinTable()); - primePvsCacheTableListingHashLoader(qInstance.getTable(queryJoin.getJoinTable()), fieldsByPvsTable, pvsesByTable, aliasOrTableName + ".", queryJoin.getJoinTable(), limitedToFieldNames); + primePvsCacheTableListingHashLoader(QContext.getQInstance().getTable(queryJoin.getJoinTable()), fieldsByPvsTable, pvsesByTable, aliasOrTableName + ".", queryJoin.getJoinTable(), limitedToFieldNames); } } @@ -474,7 +478,7 @@ public class QPossibleValueTranslator { for(QFieldMetaData field : table.getFields().values()) { - QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(field.getPossibleValueSourceName()); + QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName()); if(possibleValueSource != null && possibleValueSource.getType().equals(QPossibleValueSourceType.TABLE)) { if(limitedToFieldNames != null && !limitedToFieldNames.contains(fieldNamePrefix + field.getName())) @@ -510,7 +514,7 @@ public class QPossibleValueTranslator try { - String primaryKeyField = qInstance.getTable(tableName).getPrimaryKeyField(); + String primaryKeyField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField(); for(List page : CollectionUtils.getPages(values, 1000)) { @@ -531,7 +535,7 @@ public class QPossibleValueTranslator { if(possibleValueSource.getType().equals(QPossibleValueSourceType.TABLE)) { - QTableMetaData table = qInstance.getTable(possibleValueSource.getTableName()); + QTableMetaData table = QContext.getQInstance().getTable(possibleValueSource.getTableName()); for(String recordLabelField : CollectionUtils.nonNullList(table.getRecordLabelFields())) { QFieldMetaData field = table.getField(recordLabelField); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java index ddfa92aa..f47029dc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -282,7 +282,8 @@ public class QValueFormatter /******************************************************************************* - ** For a list of records, set their recordLabels and display values + ** For a list of records, set their recordLabels and display values - including + ** record label (e.g., from the table meta data). *******************************************************************************/ public static void setDisplayValuesInRecords(QTableMetaData table, List records) { @@ -300,6 +301,24 @@ public class QValueFormatter + /******************************************************************************* + ** For a list of records, set their recordLabels and display values + *******************************************************************************/ + public static void setDisplayValuesInRecords(Collection fields, List records) + { + if(records == null) + { + return; + } + + for(QRecord record : records) + { + setDisplayValuesInRecord(fields, record); + } + } + + + /******************************************************************************* ** For a list of records, set their display values *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 69fc7705..f82740f0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -51,7 +51,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; @@ -293,14 +292,6 @@ public class QInstanceEnricher *******************************************************************************/ private void enrichStep(QStepMetaData step) { - if(!StringUtils.hasContent(step.getName()) && step instanceof QBackendStepMetaData backendStep) - { - if(backendStep.getCode() != null && backendStep.getCode().getName() != null) - { - step.setName(backendStep.getCode().getName()); - } - } - if(!StringUtils.hasContent(step.getLabel())) { step.setLabel(nameToLabel(step.getName())); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/Aggregate.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/Aggregate.java index 5598f154..c6001695 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/Aggregate.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/Aggregate.java @@ -24,15 +24,18 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.aggregate; import java.io.Serializable; import java.util.Objects; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; /******************************************************************************* - ** + ** Define an "aggregate", e.g., to be selected in an Aggregate action. + ** Such as SUM(cost). *******************************************************************************/ public class Aggregate implements Serializable { private String fieldName; private AggregateOperator operator; + private QFieldType fieldType; @@ -55,12 +58,14 @@ public class Aggregate implements Serializable { return true; } + if(o == null || getClass() != o.getClass()) { return false; } + Aggregate aggregate = (Aggregate) o; - return Objects.equals(fieldName, aggregate.fieldName) && operator == aggregate.operator; + return Objects.equals(fieldName, aggregate.fieldName) && operator == aggregate.operator && fieldType == aggregate.fieldType; } @@ -71,7 +76,7 @@ public class Aggregate implements Serializable @Override public int hashCode() { - return Objects.hash(fieldName, operator); + return Objects.hash(fieldName, operator, fieldType); } @@ -153,4 +158,35 @@ public class Aggregate implements Serializable return (this); } + + + /******************************************************************************* + ** Getter for fieldType + *******************************************************************************/ + public QFieldType getFieldType() + { + return (this.fieldType); + } + + + + /******************************************************************************* + ** Setter for fieldType + *******************************************************************************/ + public void setFieldType(QFieldType fieldType) + { + this.fieldType = fieldType; + } + + + + /******************************************************************************* + ** Fluent setter for fieldType + *******************************************************************************/ + public Aggregate withFieldType(QFieldType fieldType) + { + this.fieldType = fieldType; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateInput.java index 6e292b4e..04459f04 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateInput.java @@ -38,6 +38,7 @@ public class AggregateInput extends AbstractTableActionInput private QQueryFilter filter; private List aggregates; private List groupBys = new ArrayList<>(); + private Integer limit; private List queryJoins = null; @@ -234,4 +235,38 @@ public class AggregateInput extends AbstractTableActionInput return (this); } + + + /******************************************************************************* + ** Getter for limit + ** + *******************************************************************************/ + public Integer getLimit() + { + return limit; + } + + + + /******************************************************************************* + ** Setter for limit + ** + *******************************************************************************/ + public void setLimit(Integer limit) + { + this.limit = limit; + } + + + + /******************************************************************************* + ** Fluent setter for limit + ** + *******************************************************************************/ + public AggregateInput withLimit(Integer limit) + { + this.limit = limit; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateOperator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateOperator.java index a83724ac..513e95c7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateOperator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateOperator.java @@ -27,9 +27,33 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.aggregate; *******************************************************************************/ public enum AggregateOperator { - COUNT, - SUM, - MIN, - MAX, - AVG + COUNT("COUNT("), + COUNT_DISTINCT("COUNT(DISTINCT "), + SUM("SUM("), + MIN("MIN("), + MAX("MAX("), + AVG("AVG("); + + private final String sqlPrefix; + + + + /******************************************************************************* + ** + *******************************************************************************/ + AggregateOperator(String sqlPrefix) + { + this.sqlPrefix = sqlPrefix; + } + + + + /******************************************************************************* + ** Getter for sqlPrefix + ** + *******************************************************************************/ + public String getSqlPrefix() + { + return sqlPrefix; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java index ce28e21c..750fc91b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java @@ -38,6 +38,17 @@ public class GroupBy implements Serializable + /******************************************************************************* + ** + *******************************************************************************/ + public GroupBy(QFieldType type, String fieldName) + { + this.type = type; + this.fieldName = fieldName; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index dd0ad81c..964720cb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -234,6 +235,12 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi { this.steps = new HashMap<>(); } + + if(!StringUtils.hasContent(step.getName())) + { + throw (new IllegalArgumentException("Attempt to add a process step without a name")); + } + this.steps.put(step.getName(), step); return (this); @@ -251,6 +258,12 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi { this.steps = new HashMap<>(); } + + if(!StringUtils.hasContent(step.getName())) + { + throw (new IllegalArgumentException("Attempt to add a process step without a name")); + } + this.steps.put(step.getName(), step); return (this); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java new file mode 100644 index 00000000..24efecdb --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java @@ -0,0 +1,318 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.columnstats; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; +import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** This is a single-step process used to provide Column Statistics. These include + ** counts per-value for a field, plus things like total count, min, max, avg, based + ** on the field type. + *******************************************************************************/ +public class ColumnStatsStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(ColumnStatsStep.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + String tableName = runBackendStepInput.getValueString("tableName"); + String fieldName = runBackendStepInput.getValueString("fieldName"); + String orderBy = runBackendStepInput.getValueString("orderBy"); + String filterJSON = runBackendStepInput.getValueString("filterJSON"); + + ///////////////////////////////////////// + // make sure user may query this table // + ///////////////////////////////////////// + PermissionsHelper.checkTablePermissionThrowing(new QueryInput().withTableName(tableName), TablePermissionSubType.READ); + + QQueryFilter filter = null; + if(StringUtils.hasContent(filterJSON)) + { + filter = JsonUtils.toObject(filterJSON, QQueryFilter.class); + + /////////////////////////////////////////////////////////////// + // ... remove any order-by that may have been in that filter // + /////////////////////////////////////////////////////////////// + filter.setOrderBys(new ArrayList<>()); + } + else + { + filter = new QQueryFilter(); + } + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + QFieldMetaData field = table.getField(fieldName); + + //////////////////////////////////////////// + // do a count query grouped by this field // + //////////////////////////////////////////// + Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL); + GroupBy groupBy = new GroupBy(field.getType(), fieldName); + + if(StringUtils.hasContent(orderBy)) + { + if(orderBy.equalsIgnoreCase("count.asc")) + { + filter.withOrderBy(new QFilterOrderByAggregate(aggregate, true)); + } + else if(orderBy.equalsIgnoreCase("count.desc")) + { + filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)); + } + else if(orderBy.equalsIgnoreCase(fieldName + ".asc")) + { + filter.withOrderBy(new QFilterOrderBy(fieldName, true)); + } + else if(orderBy.equalsIgnoreCase(fieldName + ".desc")) + { + filter.withOrderBy(new QFilterOrderBy(fieldName, false)); + } + else + { + LOG.info("Unrecognized orderBy: " + orderBy); + } + } + + /////////////////////////////////////////////////////////////////////////////////////////////////// + // always add order by to break ties. these will be the default too, if input didn't supply one // + /////////////////////////////////////////////////////////////////////////////////////////////////// + filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)); + filter.withOrderBy(new QFilterOrderBy(fieldName)); + + Integer limit = 1000; // too big? + AggregateInput aggregateInput = new AggregateInput(); + aggregateInput.withAggregate(aggregate); + aggregateInput.withGroupBy(groupBy); + aggregateInput.setTableName(tableName); + aggregateInput.setFilter(filter); + aggregateInput.setLimit(limit); + AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput); + + ArrayList valueCounts = new ArrayList<>(); + for(AggregateResult result : aggregateOutput.getResults()) + { + Serializable value = result.getGroupByValue(groupBy); + Integer count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate)); + valueCounts.add(new QRecord().withValue(fieldName, value).withValue("count", count)); + } + QFieldMetaData countField = new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS).withLabel("Count"); + + QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(); + qPossibleValueTranslator.translatePossibleValuesInRecords(table, valueCounts, null, null); + QValueFormatter.setDisplayValuesInRecords(List.of(table.getField(fieldName), countField), valueCounts); + + runBackendStepOutput.addValue("valueCounts", valueCounts); + + ///////////////////////////////////////////////////// + // now do individual statistics as a pseudo-record // + ///////////////////////////////////////////////////// + QFieldMetaData countNonNullField = new QFieldMetaData("count", QFieldType.INTEGER).withLabel("Rows with a value").withDisplayFormat(DisplayFormat.COMMAS); + QFieldMetaData countDistinctField = new QFieldMetaData("countDistinct", QFieldType.INTEGER).withLabel("Distinct values").withDisplayFormat(DisplayFormat.COMMAS); + QFieldMetaData sumField = new QFieldMetaData("sum", QFieldType.DECIMAL).withDisplayFormat(field.getDisplayFormat()); + QFieldMetaData avgField = new QFieldMetaData("average", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.DECIMAL2_COMMAS); + QFieldMetaData minField = new QFieldMetaData("min", field.getType()).withDisplayFormat(field.getDisplayFormat()); + QFieldMetaData maxField = new QFieldMetaData("max", field.getType()).withDisplayFormat(field.getDisplayFormat()); + + boolean doCountDistinct = true; + boolean doSum = true; + boolean doAvg = true; + boolean doMin = true; + boolean doMax = true; + if(field.getType().isStringLike()) + { + doSum = false; + doAvg = false; + } + if(field.getType().equals(QFieldType.BOOLEAN)) + { + doSum = false; + doAvg = false; + doMin = false; + doMax = false; + } + if(field.getType().equals(QFieldType.DATE) || field.getType().equals(QFieldType.DATE_TIME)) + { + doSum = false; + doAvg = false; // could this be done? + } + if(StringUtils.hasContent(field.getPossibleValueSourceName())) + { + doSum = false; + doAvg = false; + doMin = false; + doMax = false; + } + + if(field.getName().equals(table.getPrimaryKeyField())) + { + doSum = false; + } + + ArrayList fields = new ArrayList<>(); + fields.add(countNonNullField); + fields.add(countDistinctField); + if(doSum) + { + fields.add(sumField); + } + if(doAvg) + { + fields.add(avgField); + } + if(doMin) + { + fields.add(minField); + } + if(doMax) + { + fields.add(maxField); + } + + QRecord statsRecord = new QRecord(); + + if(valueCounts.size() < limit) + { + statsRecord.setValue(countDistinctField.getName(), valueCounts.size()); + doCountDistinct = false; + } + + ///////////////////////////////////////////////////////////////////////////////// + // just in case any of these don't fit in an integer, use decimal for them all // + ///////////////////////////////////////////////////////////////////////////////// + Aggregate countNonNullAggregate = new Aggregate(fieldName, AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL); + Aggregate countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT).withFieldType(QFieldType.DECIMAL); + Aggregate sumAggregate = new Aggregate(fieldName, AggregateOperator.SUM).withFieldType(QFieldType.DECIMAL); + Aggregate avgAggregate = new Aggregate(fieldName, AggregateOperator.AVG).withFieldType(QFieldType.DECIMAL); + Aggregate minAggregate = new Aggregate(fieldName, AggregateOperator.MIN); + Aggregate maxAggregate = new Aggregate(fieldName, AggregateOperator.MAX); + AggregateInput statsAggregateInput = new AggregateInput(); + statsAggregateInput.withAggregate(countNonNullAggregate); + if(doCountDistinct) + { + statsAggregateInput.withAggregate(countDistinctAggregate); + } + if(doSum) + { + statsAggregateInput.withAggregate(sumAggregate); + } + if(doAvg) + { + statsAggregateInput.withAggregate(avgAggregate); + } + if(doMin) + { + statsAggregateInput.withAggregate(minAggregate); + } + if(doMax) + { + statsAggregateInput.withAggregate(maxAggregate); + } + + if(CollectionUtils.nullSafeHasContents(statsAggregateInput.getAggregates())) + { + statsAggregateInput.setTableName(tableName); + filter.setOrderBys(new ArrayList<>()); + statsAggregateInput.setFilter(filter); + AggregateOutput statsAggregateOutput = new AggregateAction().execute(statsAggregateInput); + AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0); + + statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate)); + if(doCountDistinct) + { + statsRecord.setValue(countDistinctField.getName(), statsAggregateResult.getAggregateValue(countDistinctAggregate)); + } + if(doSum) + { + statsRecord.setValue(sumField.getName(), statsAggregateResult.getAggregateValue(sumAggregate)); + } + if(doAvg) + { + statsRecord.setValue(avgField.getName(), statsAggregateResult.getAggregateValue(avgAggregate)); + } + if(doMin) + { + statsRecord.setValue(minField.getName(), statsAggregateResult.getAggregateValue(minAggregate)); + } + if(doMax) + { + statsRecord.setValue(maxField.getName(), statsAggregateResult.getAggregateValue(maxAggregate)); + } + } + + QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(null); + fields.forEach(qInstanceEnricher::enrichField); + + QValueFormatter.setDisplayValuesInRecord(fields, statsRecord); + + runBackendStepOutput.addValue("statsFields", fields); + runBackendStepOutput.addValue("statsRecord", statsRecord); + } + catch(Exception e) + { + throw new QException("Error calculating stats", e); + } + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 61f803bf..0bedf9ae 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -347,11 +347,11 @@ class QInstanceValidatorTest extends BaseTest @Test public void test_validateProcessStepWithEmptyName() { - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // these used to be an assertion failure - but enricher now sets a default name for backend steps w/ a code name // - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - assertValidationSuccess((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).getStepList().get(0).setName("")); - assertValidationSuccess((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().get(1).setName(null)); + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).getStepList().get(0).setName(""), + "Missing name for a step"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().get(1).setName(null), + "Missing name for a step"); } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java index f5eaf881..b7ba77ea 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java @@ -85,6 +85,11 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega sql += " ORDER BY " + makeOrderByClause(table, filter.getOrderBys(), joinsContext); } + if(aggregateInput.getLimit() != null) + { + sql += " LIMIT " + aggregateInput.getLimit(); + } + // todo sql customization - can edit sql and/or param list AggregateOutput rs = new AggregateOutput(); @@ -114,9 +119,18 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(aggregate.getFieldName()); QFieldMetaData field = fieldAndTableNameOrAlias.field(); - if(field.getType().equals(QFieldType.INTEGER) && aggregate.getOperator().equals(AggregateOperator.AVG)) + QFieldType fieldType = aggregate.getFieldType(); + if(fieldType == null) { - field = new QFieldMetaData().withType(QFieldType.DECIMAL); + if(field.getType().equals(QFieldType.INTEGER) && (aggregate.getOperator().equals(AggregateOperator.AVG))) + { + fieldType = QFieldType.DECIMAL; + } + } + + if(fieldType != null) + { + field = new QFieldMetaData().withType(fieldType); } Serializable value = getFieldValueFromResultSet(field, resultSet, selectionIndex++); @@ -155,7 +169,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega for(Aggregate aggregate : aggregateInput.getAggregates()) { JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(aggregate.getFieldName()); - rs.add(aggregate.getOperator() + "(" + escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(fieldAndTableNameOrAlias.field())) + ")"); + rs.add(aggregate.getOperator().getSqlPrefix() + escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(fieldAndTableNameOrAlias.field())) + ")"); } return (rs); }