diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java index 84f6f9fc..433e19ac 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java @@ -33,6 +33,8 @@ import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; @@ -42,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOu import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult; import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -49,6 +52,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -59,6 +63,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public class TableStatsStep implements BackendStep { + private static final QLogger LOG = QLogger.getLogger(TableStatsStep.class); + + /******************************************************************************* ** @@ -70,6 +77,7 @@ public class TableStatsStep implements BackendStep { String tableName = runBackendStepInput.getValueString("tableName"); String fieldName = runBackendStepInput.getValueString("fieldName"); + String orderBy = runBackendStepInput.getValueString("orderBy"); String filterJSON = runBackendStepInput.getValueString("filterJSON"); ///////////////////////////////////////// @@ -95,15 +103,48 @@ public class TableStatsStep implements BackendStep QTableMetaData table = QContext.getQInstance().getTable(tableName); QFieldMetaData field = table.getField(fieldName); - Aggregate aggregate = new Aggregate(fieldName, AggregateOperator.COUNT); + //////////////////////////////////////////// + // do a count query grouped by this field // + //////////////////////////////////////////// + Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT); GroupBy groupBy = new GroupBy(field.getType(), fieldName); + if(StringUtils.hasContent(orderBy)) + { + if(orderBy.equalsIgnoreCase("count.asc")) + { + filter.withOrderBy(new QFilterOrderByAggregate(aggregate, true)); + } + else if(orderBy.equalsIgnoreCase("count.desc")) + { + filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)); + } + else if(orderBy.equalsIgnoreCase(fieldName + ".asc")) + { + filter.withOrderBy(new QFilterOrderBy(fieldName, true)); + } + else if(orderBy.equalsIgnoreCase(fieldName + ".desc")) + { + filter.withOrderBy(new QFilterOrderBy(fieldName, false)); + } + else + { + LOG.info("Unrecognized orderBy: " + orderBy); + } + } + + /////////////////////////////////////////////////////////////////////////////////////////////////// + // always add order by to break ties. these will be the default too, if input didn't supply one // + /////////////////////////////////////////////////////////////////////////////////////////////////// + filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)); + filter.withOrderBy(new QFilterOrderBy(fieldName)); + Integer limit = 1000; // too big? AggregateInput aggregateInput = new AggregateInput(); aggregateInput.withAggregate(aggregate); aggregateInput.withGroupBy(groupBy); aggregateInput.setTableName(tableName); - aggregateInput.setFilter(filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false))); + aggregateInput.setFilter(filter); aggregateInput.setLimit(limit); AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput); @@ -122,21 +163,141 @@ public class TableStatsStep implements BackendStep runBackendStepOutput.addValue("valueCounts", valueCounts); + ///////////////////////////////////////////////////// + // now do individual statistics as a pseudo-record // + ///////////////////////////////////////////////////// + QFieldMetaData countNonNullField = new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS); + QFieldMetaData countDistinctField = new QFieldMetaData("countDistinct", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS); + QFieldMetaData sumField = new QFieldMetaData("sum", QFieldType.DECIMAL).withDisplayFormat(field.getDisplayFormat()); + QFieldMetaData avgField = new QFieldMetaData("average", QFieldType.DECIMAL).withDisplayFormat(field.getDisplayFormat()); + QFieldMetaData minField = new QFieldMetaData("min", field.getType()).withDisplayFormat(field.getDisplayFormat()); + QFieldMetaData maxField = new QFieldMetaData("max", field.getType()).withDisplayFormat(field.getDisplayFormat()); + + boolean doCountDistinct = true; + boolean doSum = true; + boolean doAvg = true; + boolean doMin = true; + boolean doMax = true; + if(field.getType().isStringLike()) + { + doSum = false; + doAvg = false; + } + if(field.getType().equals(QFieldType.BOOLEAN)) + { + doSum = false; + doAvg = false; + doMin = false; + doMax = false; + } + if(field.getType().equals(QFieldType.DATE) || field.getType().equals(QFieldType.DATE_TIME)) + { + doSum = false; + doAvg = false; // could this be done? + } + if(StringUtils.hasContent(field.getPossibleValueSourceName())) + { + doSum = false; + doAvg = false; + doMin = false; + doMax = false; + } + + ArrayList fields = new ArrayList<>(); + fields.add(countNonNullField); + fields.add(countDistinctField); + if(doSum) + { + fields.add(sumField); + } + if(doAvg) + { + fields.add(avgField); + } + if(doMin) + { + fields.add(minField); + } + if(doMax) + { + fields.add(maxField); + } + + QRecord statsRecord = new QRecord(); + if(valueCounts.size() < limit) { - runBackendStepOutput.addValue("countDistinct", valueCounts.size()); + statsRecord.setValue(countDistinctField.getName(), valueCounts.size()); + doCountDistinct = false; } - else + + Aggregate countNonNullAggregate = new Aggregate(fieldName, AggregateOperator.COUNT); + Aggregate countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT); + Aggregate sumAggregate = new Aggregate(fieldName, AggregateOperator.SUM); + Aggregate avgAggregate = new Aggregate(fieldName, AggregateOperator.AVG); + Aggregate minAggregate = new Aggregate(fieldName, AggregateOperator.MIN); + Aggregate maxAggregate = new Aggregate(fieldName, AggregateOperator.MAX); + AggregateInput statsAggregateInput = new AggregateInput(); + statsAggregateInput.withAggregate(countNonNullAggregate); + if(doCountDistinct) { - Aggregate countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT); - AggregateInput countDistinctAggregateInput = new AggregateInput(); - countDistinctAggregateInput.withAggregate(countDistinctAggregate); - countDistinctAggregateInput.setTableName(tableName); - countDistinctAggregateInput.setFilter(filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false))); - AggregateOutput countDistinctAggregateOutput = new AggregateAction().execute(countDistinctAggregateInput); - AggregateResult countDistinctAggregateResult = countDistinctAggregateOutput.getResults().get(0); - runBackendStepOutput.addValue("countDistinct", countDistinctAggregateResult.getAggregateValue(countDistinctAggregate)); + statsAggregateInput.withAggregate(countDistinctAggregate); } + if(doSum) + { + statsAggregateInput.withAggregate(sumAggregate); + } + if(doAvg) + { + statsAggregateInput.withAggregate(avgAggregate); + } + if(doMin) + { + statsAggregateInput.withAggregate(minAggregate); + } + if(doMax) + { + statsAggregateInput.withAggregate(maxAggregate); + } + + if(CollectionUtils.nullSafeHasContents(statsAggregateInput.getAggregates())) + { + statsAggregateInput.setTableName(tableName); + filter.setOrderBys(new ArrayList<>()); + statsAggregateInput.setFilter(filter); + AggregateOutput statsAggregateOutput = new AggregateAction().execute(statsAggregateInput); + AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0); + + statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate)); + if(doCountDistinct) + { + statsRecord.setValue(countDistinctField.getName(), statsAggregateResult.getAggregateValue(countDistinctAggregate)); + } + if(doSum) + { + statsRecord.setValue(sumField.getName(), statsAggregateResult.getAggregateValue(sumAggregate)); + } + if(doAvg) + { + statsRecord.setValue(avgField.getName(), statsAggregateResult.getAggregateValue(avgAggregate)); + } + if(doMin) + { + statsRecord.setValue(minField.getName(), statsAggregateResult.getAggregateValue(minAggregate)); + } + if(doMax) + { + statsRecord.setValue(maxField.getName(), statsAggregateResult.getAggregateValue(maxAggregate)); + } + } + + QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(null); + fields.forEach(qInstanceEnricher::enrichField); + + QValueFormatter.setDisplayValuesInRecord(fields, statsRecord); + + runBackendStepOutput.addValue("statsFields", fields); + runBackendStepOutput.addValue("statsRecord", statsRecord); } catch(Exception e) { diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java index fec2d3a1..ab4a1222 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java @@ -119,7 +119,12 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(aggregate.getFieldName()); QFieldMetaData field = fieldAndTableNameOrAlias.field(); - if(field.getType().equals(QFieldType.INTEGER) && aggregate.getOperator().equals(AggregateOperator.AVG)) + if(field.getType().equals(QFieldType.INTEGER) && (aggregate.getOperator().equals(AggregateOperator.AVG) || aggregate.getOperator().equals(AggregateOperator.SUM))) + { + field = new QFieldMetaData().withType(QFieldType.DECIMAL); + } + + if(aggregate.getOperator().equals(AggregateOperator.COUNT) || aggregate.getOperator().equals(AggregateOperator.COUNT_DISTINCT)) { field = new QFieldMetaData().withType(QFieldType.DECIMAL); }