More stats & aggregates

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java

@@ -33,6 +33,8 @@ import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
@@ -42,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOu
 import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
 import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -49,6 +52,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@@ -59,6 +63,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
  *******************************************************************************/
 public class TableStatsStep implements BackendStep
 {
+   private static final QLogger LOG = QLogger.getLogger(TableStatsStep.class);
+
+
 
    /*******************************************************************************
     **
@@ -70,6 +77,7 @@ public class TableStatsStep implements BackendStep
       {
          String tableName  = runBackendStepInput.getValueString("tableName");
          String fieldName  = runBackendStepInput.getValueString("fieldName");
+         String orderBy    = runBackendStepInput.getValueString("orderBy");
          String filterJSON = runBackendStepInput.getValueString("filterJSON");
 
          /////////////////////////////////////////
@@ -95,15 +103,48 @@ public class TableStatsStep implements BackendStep
          QTableMetaData table = QContext.getQInstance().getTable(tableName);
          QFieldMetaData field = table.getField(fieldName);
 
-         Aggregate aggregate = new Aggregate(fieldName, AggregateOperator.COUNT);
+         ////////////////////////////////////////////
+         // do a count query grouped by this field //
+         ////////////////////////////////////////////
+         Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT);
          GroupBy   groupBy   = new GroupBy(field.getType(), fieldName);
 
+         if(StringUtils.hasContent(orderBy))
+         {
+            if(orderBy.equalsIgnoreCase("count.asc"))
+            {
+               filter.withOrderBy(new QFilterOrderByAggregate(aggregate, true));
+            }
+            else if(orderBy.equalsIgnoreCase("count.desc"))
+            {
+               filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false));
+            }
+            else if(orderBy.equalsIgnoreCase(fieldName + ".asc"))
+            {
+               filter.withOrderBy(new QFilterOrderBy(fieldName, true));
+            }
+            else if(orderBy.equalsIgnoreCase(fieldName + ".desc"))
+            {
+               filter.withOrderBy(new QFilterOrderBy(fieldName, false));
+            }
+            else
+            {
+               LOG.info("Unrecognized orderBy: " + orderBy);
+            }
+         }
+
+         ///////////////////////////////////////////////////////////////////////////////////////////////////
+         // always add order by to break ties.  these will be the default too, if input didn't supply one //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////
+         filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false));
+         filter.withOrderBy(new QFilterOrderBy(fieldName));
+
          Integer        limit          = 1000; // too big?
          AggregateInput aggregateInput = new AggregateInput();
          aggregateInput.withAggregate(aggregate);
          aggregateInput.withGroupBy(groupBy);
          aggregateInput.setTableName(tableName);
-         aggregateInput.setFilter(filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)));
+         aggregateInput.setFilter(filter);
          aggregateInput.setLimit(limit);
          AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
 
@@ -122,21 +163,141 @@ public class TableStatsStep implements BackendStep
 
          runBackendStepOutput.addValue("valueCounts", valueCounts);
 
+         /////////////////////////////////////////////////////
+         // now do individual statistics as a pseudo-record //
+         /////////////////////////////////////////////////////
+         QFieldMetaData countNonNullField  = new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS);
+         QFieldMetaData countDistinctField = new QFieldMetaData("countDistinct", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS);
+         QFieldMetaData sumField           = new QFieldMetaData("sum", QFieldType.DECIMAL).withDisplayFormat(field.getDisplayFormat());
+         QFieldMetaData avgField           = new QFieldMetaData("average", QFieldType.DECIMAL).withDisplayFormat(field.getDisplayFormat());
+         QFieldMetaData minField           = new QFieldMetaData("min", field.getType()).withDisplayFormat(field.getDisplayFormat());
+         QFieldMetaData maxField           = new QFieldMetaData("max", field.getType()).withDisplayFormat(field.getDisplayFormat());
+
+         boolean doCountDistinct = true;
+         boolean doSum           = true;
+         boolean doAvg           = true;
+         boolean doMin           = true;
+         boolean doMax           = true;
+         if(field.getType().isStringLike())
+         {
+            doSum = false;
+            doAvg = false;
+         }
+         if(field.getType().equals(QFieldType.BOOLEAN))
+         {
+            doSum = false;
+            doAvg = false;
+            doMin = false;
+            doMax = false;
+         }
+         if(field.getType().equals(QFieldType.DATE) || field.getType().equals(QFieldType.DATE_TIME))
+         {
+            doSum = false;
+            doAvg = false; // could this be done?
+         }
+         if(StringUtils.hasContent(field.getPossibleValueSourceName()))
+         {
+            doSum = false;
+            doAvg = false;
+            doMin = false;
+            doMax = false;
+         }
+
+         ArrayList<QFieldMetaData> fields = new ArrayList<>();
+         fields.add(countNonNullField);
+         fields.add(countDistinctField);
+         if(doSum)
+         {
+            fields.add(sumField);
+         }
+         if(doAvg)
+         {
+            fields.add(avgField);
+         }
+         if(doMin)
+         {
+            fields.add(minField);
+         }
+         if(doMax)
+         {
+            fields.add(maxField);
+         }
+
+         QRecord statsRecord = new QRecord();
+
          if(valueCounts.size() < limit)
          {
-            runBackendStepOutput.addValue("countDistinct", valueCounts.size());
+            statsRecord.setValue(countDistinctField.getName(), valueCounts.size());
+            doCountDistinct = false;
          }
-         else
+
-         {
+         Aggregate      countNonNullAggregate  = new Aggregate(fieldName, AggregateOperator.COUNT);
          Aggregate      countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT);
-            AggregateInput countDistinctAggregateInput = new AggregateInput();
+         Aggregate      sumAggregate           = new Aggregate(fieldName, AggregateOperator.SUM);
-            countDistinctAggregateInput.withAggregate(countDistinctAggregate);
+         Aggregate      avgAggregate           = new Aggregate(fieldName, AggregateOperator.AVG);
-            countDistinctAggregateInput.setTableName(tableName);
+         Aggregate      minAggregate           = new Aggregate(fieldName, AggregateOperator.MIN);
-            countDistinctAggregateInput.setFilter(filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)));
+         Aggregate      maxAggregate           = new Aggregate(fieldName, AggregateOperator.MAX);
-            AggregateOutput countDistinctAggregateOutput = new AggregateAction().execute(countDistinctAggregateInput);
+         AggregateInput statsAggregateInput    = new AggregateInput();
-            AggregateResult countDistinctAggregateResult = countDistinctAggregateOutput.getResults().get(0);
+         statsAggregateInput.withAggregate(countNonNullAggregate);
-            runBackendStepOutput.addValue("countDistinct", countDistinctAggregateResult.getAggregateValue(countDistinctAggregate));
+         if(doCountDistinct)
+         {
+            statsAggregateInput.withAggregate(countDistinctAggregate);
          }
+         if(doSum)
+         {
+            statsAggregateInput.withAggregate(sumAggregate);
+         }
+         if(doAvg)
+         {
+            statsAggregateInput.withAggregate(avgAggregate);
+         }
+         if(doMin)
+         {
+            statsAggregateInput.withAggregate(minAggregate);
+         }
+         if(doMax)
+         {
+            statsAggregateInput.withAggregate(maxAggregate);
+         }
+
+         if(CollectionUtils.nullSafeHasContents(statsAggregateInput.getAggregates()))
+         {
+            statsAggregateInput.setTableName(tableName);
+            filter.setOrderBys(new ArrayList<>());
+            statsAggregateInput.setFilter(filter);
+            AggregateOutput statsAggregateOutput = new AggregateAction().execute(statsAggregateInput);
+            AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0);
+
+            statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate));
+            if(doCountDistinct)
+            {
+               statsRecord.setValue(countDistinctField.getName(), statsAggregateResult.getAggregateValue(countDistinctAggregate));
+            }
+            if(doSum)
+            {
+               statsRecord.setValue(sumField.getName(), statsAggregateResult.getAggregateValue(sumAggregate));
+            }
+            if(doAvg)
+            {
+               statsRecord.setValue(avgField.getName(), statsAggregateResult.getAggregateValue(avgAggregate));
+            }
+            if(doMin)
+            {
+               statsRecord.setValue(minField.getName(), statsAggregateResult.getAggregateValue(minAggregate));
+            }
+            if(doMax)
+            {
+               statsRecord.setValue(maxField.getName(), statsAggregateResult.getAggregateValue(maxAggregate));
+            }
+         }
+
+         QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(null);
+         fields.forEach(qInstanceEnricher::enrichField);
+
+         QValueFormatter.setDisplayValuesInRecord(fields, statsRecord);
+
+         runBackendStepOutput.addValue("statsFields", fields);
+         runBackendStepOutput.addValue("statsRecord", statsRecord);
       }
       catch(Exception e)
       {

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java

@@ -119,7 +119,12 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
                      JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(aggregate.getFieldName());
                      QFieldMetaData                        field                    = fieldAndTableNameOrAlias.field();
 
-                     if(field.getType().equals(QFieldType.INTEGER) && aggregate.getOperator().equals(AggregateOperator.AVG))
+                     if(field.getType().equals(QFieldType.INTEGER) && (aggregate.getOperator().equals(AggregateOperator.AVG) || aggregate.getOperator().equals(AggregateOperator.SUM)))
+                     {
+                        field = new QFieldMetaData().withType(QFieldType.DECIMAL);
+                     }
+
+                     if(aggregate.getOperator().equals(AggregateOperator.COUNT) || aggregate.getOperator().equals(AggregateOperator.COUNT_DISTINCT))
                      {
                         field = new QFieldMetaData().withType(QFieldType.DECIMAL);
                      }