Merge pull request #13 from Kingsrook/feature/column-stats

Feature/column stats

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java

@@ -436,7 +436,7 @@ public class PermissionsHelper
             }
          }
 
-         if(hasPermission(actionInput.getSession(), permissionBaseName, effectivePermissionSubType))
+         if(hasPermission(QContext.getQSession(), permissionBaseName, effectivePermissionSubType))
          {
             return (PermissionCheckResult.ALLOW);
          }
@@ -534,9 +534,9 @@ public class PermissionsHelper
          return;
       }
 
-      if(!hasPermission(actionInput.getSession(), permissionBaseName, effectivePermissionSubType))
+      if(!hasPermission(QContext.getQSession(), permissionBaseName, effectivePermissionSubType))
       {
-         // LOG.debug("Throwing permission denied for: " + getPermissionName(permissionBaseName, effectivePermissionSubType) + " for " + actionInput.getSession().getUser());
+         // LOG.debug("Throwing permission denied for: " + getPermissionName(permissionBaseName, effectivePermissionSubType) + " for " + QContext.getQSession().getUser());
          throw (new QPermissionDeniedException("Permission denied."));
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java

@@ -34,6 +34,7 @@ import java.util.Objects;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QValueException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
@@ -67,28 +68,31 @@ public class QPossibleValueTranslator
 {
    private static final QLogger LOG = QLogger.getLogger(QPossibleValueTranslator.class);
 
-   private final QInstance qInstance;
-   private final QSession  session;
-
    ///////////////////////////////////////////////////////
    // top-level keys are pvsNames (not table names)     //
    // 2nd-level keys are pkey values from the PVS table //
    ///////////////////////////////////////////////////////
-   private Map<String, Map<Serializable, String>> possibleValueCache;
+   private Map<String, Map<Serializable, String>> possibleValueCache = new HashMap<>();
 
    // todo not commit - remove instance & session - use Context
 
 
 
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public QPossibleValueTranslator()
+   {
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    public QPossibleValueTranslator(QInstance qInstance, QSession session)
    {
-      this.qInstance = qInstance;
-      this.session = session;
-
-      this.possibleValueCache = new HashMap<>();
    }
 
 
@@ -141,7 +145,7 @@ public class QPossibleValueTranslator
             {
                try
                {
-                  QTableMetaData joinTable = qInstance.getTable(queryJoin.getJoinTable());
+                  QTableMetaData joinTable = QContext.getQInstance().getTable(queryJoin.getJoinTable());
                   for(QFieldMetaData field : joinTable.getFields().values())
                   {
                      String joinFieldName = Objects.requireNonNullElse(queryJoin.getAlias(), joinTable.getName()) + "." + field.getName();
@@ -152,7 +156,7 @@ public class QPossibleValueTranslator
                            ///////////////////////////////////////////////
                            // avoid circling-back upon the source table //
                            ///////////////////////////////////////////////
-                           QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(field.getPossibleValueSourceName());
+                           QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName());
                            if(QPossibleValueSourceType.TABLE.equals(possibleValueSource.getType()) && table.getName().equals(possibleValueSource.getTableName()))
                            {
                               continue;
@@ -212,7 +216,7 @@ public class QPossibleValueTranslator
     *******************************************************************************/
    public String translatePossibleValue(QFieldMetaData field, Serializable value)
    {
-      QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(field.getPossibleValueSourceName());
+      QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName());
       if(possibleValueSource == null)
       {
          LOG.error("Missing possible value source named [" + field.getPossibleValueSourceName() + "] when formatting value for field [" + field.getName() + "]");
@@ -414,7 +418,7 @@ public class QPossibleValueTranslator
          if(queryJoin.getSelect())
          {
             String aliasOrTableName = Objects.requireNonNullElse(queryJoin.getAlias(), queryJoin.getJoinTable());
-            primePvsCacheTableListingHashLoader(qInstance.getTable(queryJoin.getJoinTable()), fieldsByPvsTable, pvsesByTable, aliasOrTableName + ".", queryJoin.getJoinTable(), limitedToFieldNames);
+            primePvsCacheTableListingHashLoader(QContext.getQInstance().getTable(queryJoin.getJoinTable()), fieldsByPvsTable, pvsesByTable, aliasOrTableName + ".", queryJoin.getJoinTable(), limitedToFieldNames);
          }
       }
 
@@ -474,7 +478,7 @@ public class QPossibleValueTranslator
    {
       for(QFieldMetaData field : table.getFields().values())
       {
-         QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(field.getPossibleValueSourceName());
+         QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName());
          if(possibleValueSource != null && possibleValueSource.getType().equals(QPossibleValueSourceType.TABLE))
          {
             if(limitedToFieldNames != null && !limitedToFieldNames.contains(fieldNamePrefix + field.getName()))
@@ -510,7 +514,7 @@ public class QPossibleValueTranslator
 
       try
       {
-         String primaryKeyField = qInstance.getTable(tableName).getPrimaryKeyField();
+         String primaryKeyField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField();
 
          for(List<Serializable> page : CollectionUtils.getPages(values, 1000))
          {
@@ -531,7 +535,7 @@ public class QPossibleValueTranslator
             {
                if(possibleValueSource.getType().equals(QPossibleValueSourceType.TABLE))
                {
-                  QTableMetaData table = qInstance.getTable(possibleValueSource.getTableName());
+                  QTableMetaData table = QContext.getQInstance().getTable(possibleValueSource.getTableName());
                   for(String recordLabelField : CollectionUtils.nonNullList(table.getRecordLabelFields()))
                   {
                      QFieldMetaData field = table.getField(recordLabelField);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java

@@ -282,7 +282,8 @@ public class QValueFormatter
 
 
    /*******************************************************************************
-    ** For a list of records, set their recordLabels and display values
+    ** For a list of records, set their recordLabels and display values - including
+    ** record label (e.g., from the table meta data).
     *******************************************************************************/
    public static void setDisplayValuesInRecords(QTableMetaData table, List<QRecord> records)
    {
@@ -300,6 +301,24 @@ public class QValueFormatter
 
 
 
+   /*******************************************************************************
+    ** For a list of records, set their recordLabels and display values
+    *******************************************************************************/
+   public static void setDisplayValuesInRecords(Collection<QFieldMetaData> fields, List<QRecord> records)
+   {
+      if(records == null)
+      {
+         return;
+      }
+
+      for(QRecord record : records)
+      {
+         setDisplayValuesInRecord(fields, record);
+      }
+   }
+
+
+
    /*******************************************************************************
     ** For a list of records, set their display values
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -51,7 +51,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
@@ -293,14 +292,6 @@ public class QInstanceEnricher
     *******************************************************************************/
    private void enrichStep(QStepMetaData step)
    {
-      if(!StringUtils.hasContent(step.getName()) && step instanceof QBackendStepMetaData backendStep)
-      {
-         if(backendStep.getCode() != null && backendStep.getCode().getName() != null)
-         {
-            step.setName(backendStep.getCode().getName());
-         }
-      }
-
       if(!StringUtils.hasContent(step.getLabel()))
       {
          step.setLabel(nameToLabel(step.getName()));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/Aggregate.java

@@ -24,15 +24,18 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.aggregate;
 
 import java.io.Serializable;
 import java.util.Objects;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 
 
 /*******************************************************************************
- **
+ ** Define an "aggregate", e.g., to be selected in an Aggregate action.
+ ** Such as SUM(cost).
  *******************************************************************************/
 public class Aggregate implements Serializable
 {
    private String            fieldName;
    private AggregateOperator operator;
+   private QFieldType        fieldType;
 
 
 
@@ -55,12 +58,14 @@ public class Aggregate implements Serializable
       {
          return true;
       }
+
       if(o == null || getClass() != o.getClass())
       {
          return false;
       }
+
       Aggregate aggregate = (Aggregate) o;
-      return Objects.equals(fieldName, aggregate.fieldName) && operator == aggregate.operator;
+      return Objects.equals(fieldName, aggregate.fieldName) && operator == aggregate.operator && fieldType == aggregate.fieldType;
    }
 
 
@@ -71,7 +76,7 @@ public class Aggregate implements Serializable
    @Override
    public int hashCode()
    {
-      return Objects.hash(fieldName, operator);
+      return Objects.hash(fieldName, operator, fieldType);
    }
 
 
@@ -153,4 +158,35 @@ public class Aggregate implements Serializable
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for fieldType
+    *******************************************************************************/
+   public QFieldType getFieldType()
+   {
+      return (this.fieldType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldType
+    *******************************************************************************/
+   public void setFieldType(QFieldType fieldType)
+   {
+      this.fieldType = fieldType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldType
+    *******************************************************************************/
+   public Aggregate withFieldType(QFieldType fieldType)
+   {
+      this.fieldType = fieldType;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateInput.java

@@ -38,6 +38,7 @@ public class AggregateInput extends AbstractTableActionInput
    private QQueryFilter    filter;
    private List<Aggregate> aggregates;
    private List<GroupBy>   groupBys = new ArrayList<>();
+   private Integer         limit;
 
    private List<QueryJoin> queryJoins = null;
 
@@ -234,4 +235,38 @@ public class AggregateInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for limit
+    **
+    *******************************************************************************/
+   public Integer getLimit()
+   {
+      return limit;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for limit
+    **
+    *******************************************************************************/
+   public void setLimit(Integer limit)
+   {
+      this.limit = limit;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for limit
+    **
+    *******************************************************************************/
+   public AggregateInput withLimit(Integer limit)
+   {
+      this.limit = limit;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateOperator.java

@@ -27,9 +27,33 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.aggregate;
  *******************************************************************************/
 public enum AggregateOperator
 {
-   COUNT,
+   COUNT("COUNT("),
-   SUM,
+   COUNT_DISTINCT("COUNT(DISTINCT "),
-   MIN,
+   SUM("SUM("),
-   MAX,
+   MIN("MIN("),
-   AVG
+   MAX("MAX("),
+   AVG("AVG(");
+
+   private final String sqlPrefix;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   AggregateOperator(String sqlPrefix)
+   {
+      this.sqlPrefix = sqlPrefix;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for sqlPrefix
+    **
+    *******************************************************************************/
+   public String getSqlPrefix()
+   {
+      return sqlPrefix;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java

@@ -38,6 +38,17 @@ public class GroupBy implements Serializable
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public GroupBy(QFieldType type, String fieldName)
+   {
+      this.type = type;
+      this.fieldName = fieldName;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java

@@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
 import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
@@ -234,6 +235,12 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
       {
          this.steps = new HashMap<>();
       }
+
+      if(!StringUtils.hasContent(step.getName()))
+      {
+         throw (new IllegalArgumentException("Attempt to add a process step without a name"));
+      }
+
       this.steps.put(step.getName(), step);
 
       return (this);
@@ -251,6 +258,12 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
       {
          this.steps = new HashMap<>();
       }
+
+      if(!StringUtils.hasContent(step.getName()))
+      {
+         throw (new IllegalArgumentException("Attempt to add a process step without a name"));
+      }
+
       this.steps.put(step.getName(), step);
 
       return (this);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java

@@ -0,0 +1,318 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
+import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** This is a single-step process used to provide Column Statistics.  These include
+ ** counts per-value for a field, plus things like total count, min, max, avg, based
+ ** on the field type.
+ *******************************************************************************/
+public class ColumnStatsStep implements BackendStep
+{
+   private static final QLogger LOG = QLogger.getLogger(ColumnStatsStep.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      try
+      {
+         String tableName  = runBackendStepInput.getValueString("tableName");
+         String fieldName  = runBackendStepInput.getValueString("fieldName");
+         String orderBy    = runBackendStepInput.getValueString("orderBy");
+         String filterJSON = runBackendStepInput.getValueString("filterJSON");
+
+         /////////////////////////////////////////
+         // make sure user may query this table //
+         /////////////////////////////////////////
+         PermissionsHelper.checkTablePermissionThrowing(new QueryInput().withTableName(tableName), TablePermissionSubType.READ);
+
+         QQueryFilter filter = null;
+         if(StringUtils.hasContent(filterJSON))
+         {
+            filter = JsonUtils.toObject(filterJSON, QQueryFilter.class);
+
+            ///////////////////////////////////////////////////////////////
+            // ... remove any order-by that may have been in that filter //
+            ///////////////////////////////////////////////////////////////
+            filter.setOrderBys(new ArrayList<>());
+         }
+         else
+         {
+            filter = new QQueryFilter();
+         }
+
+         QTableMetaData table = QContext.getQInstance().getTable(tableName);
+         QFieldMetaData field = table.getField(fieldName);
+
+         ////////////////////////////////////////////
+         // do a count query grouped by this field //
+         ////////////////////////////////////////////
+         Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
+         GroupBy   groupBy   = new GroupBy(field.getType(), fieldName);
+
+         if(StringUtils.hasContent(orderBy))
+         {
+            if(orderBy.equalsIgnoreCase("count.asc"))
+            {
+               filter.withOrderBy(new QFilterOrderByAggregate(aggregate, true));
+            }
+            else if(orderBy.equalsIgnoreCase("count.desc"))
+            {
+               filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false));
+            }
+            else if(orderBy.equalsIgnoreCase(fieldName + ".asc"))
+            {
+               filter.withOrderBy(new QFilterOrderBy(fieldName, true));
+            }
+            else if(orderBy.equalsIgnoreCase(fieldName + ".desc"))
+            {
+               filter.withOrderBy(new QFilterOrderBy(fieldName, false));
+            }
+            else
+            {
+               LOG.info("Unrecognized orderBy: " + orderBy);
+            }
+         }
+
+         ///////////////////////////////////////////////////////////////////////////////////////////////////
+         // always add order by to break ties.  these will be the default too, if input didn't supply one //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////
+         filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false));
+         filter.withOrderBy(new QFilterOrderBy(fieldName));
+
+         Integer        limit          = 1000; // too big?
+         AggregateInput aggregateInput = new AggregateInput();
+         aggregateInput.withAggregate(aggregate);
+         aggregateInput.withGroupBy(groupBy);
+         aggregateInput.setTableName(tableName);
+         aggregateInput.setFilter(filter);
+         aggregateInput.setLimit(limit);
+         AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+
+         ArrayList<QRecord> valueCounts = new ArrayList<>();
+         for(AggregateResult result : aggregateOutput.getResults())
+         {
+            Serializable value = result.getGroupByValue(groupBy);
+            Integer      count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate));
+            valueCounts.add(new QRecord().withValue(fieldName, value).withValue("count", count));
+         }
+         QFieldMetaData countField = new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS).withLabel("Count");
+
+         QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator();
+         qPossibleValueTranslator.translatePossibleValuesInRecords(table, valueCounts, null, null);
+         QValueFormatter.setDisplayValuesInRecords(List.of(table.getField(fieldName), countField), valueCounts);
+
+         runBackendStepOutput.addValue("valueCounts", valueCounts);
+
+         /////////////////////////////////////////////////////
+         // now do individual statistics as a pseudo-record //
+         /////////////////////////////////////////////////////
+         QFieldMetaData countNonNullField  = new QFieldMetaData("count", QFieldType.INTEGER).withLabel("Rows with a value").withDisplayFormat(DisplayFormat.COMMAS);
+         QFieldMetaData countDistinctField = new QFieldMetaData("countDistinct", QFieldType.INTEGER).withLabel("Distinct values").withDisplayFormat(DisplayFormat.COMMAS);
+         QFieldMetaData sumField           = new QFieldMetaData("sum", QFieldType.DECIMAL).withDisplayFormat(field.getDisplayFormat());
+         QFieldMetaData avgField           = new QFieldMetaData("average", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.DECIMAL2_COMMAS);
+         QFieldMetaData minField           = new QFieldMetaData("min", field.getType()).withDisplayFormat(field.getDisplayFormat());
+         QFieldMetaData maxField           = new QFieldMetaData("max", field.getType()).withDisplayFormat(field.getDisplayFormat());
+
+         boolean doCountDistinct = true;
+         boolean doSum           = true;
+         boolean doAvg           = true;
+         boolean doMin           = true;
+         boolean doMax           = true;
+         if(field.getType().isStringLike())
+         {
+            doSum = false;
+            doAvg = false;
+         }
+         if(field.getType().equals(QFieldType.BOOLEAN))
+         {
+            doSum = false;
+            doAvg = false;
+            doMin = false;
+            doMax = false;
+         }
+         if(field.getType().equals(QFieldType.DATE) || field.getType().equals(QFieldType.DATE_TIME))
+         {
+            doSum = false;
+            doAvg = false; // could this be done?
+         }
+         if(StringUtils.hasContent(field.getPossibleValueSourceName()))
+         {
+            doSum = false;
+            doAvg = false;
+            doMin = false;
+            doMax = false;
+         }
+
+         if(field.getName().equals(table.getPrimaryKeyField()))
+         {
+            doSum = false;
+         }
+
+         ArrayList<QFieldMetaData> fields = new ArrayList<>();
+         fields.add(countNonNullField);
+         fields.add(countDistinctField);
+         if(doSum)
+         {
+            fields.add(sumField);
+         }
+         if(doAvg)
+         {
+            fields.add(avgField);
+         }
+         if(doMin)
+         {
+            fields.add(minField);
+         }
+         if(doMax)
+         {
+            fields.add(maxField);
+         }
+
+         QRecord statsRecord = new QRecord();
+
+         if(valueCounts.size() < limit)
+         {
+            statsRecord.setValue(countDistinctField.getName(), valueCounts.size());
+            doCountDistinct = false;
+         }
+
+         /////////////////////////////////////////////////////////////////////////////////
+         // just in case any of these don't fit in an integer, use decimal for them all //
+         /////////////////////////////////////////////////////////////////////////////////
+         Aggregate      countNonNullAggregate  = new Aggregate(fieldName, AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
+         Aggregate      countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT).withFieldType(QFieldType.DECIMAL);
+         Aggregate      sumAggregate           = new Aggregate(fieldName, AggregateOperator.SUM).withFieldType(QFieldType.DECIMAL);
+         Aggregate      avgAggregate           = new Aggregate(fieldName, AggregateOperator.AVG).withFieldType(QFieldType.DECIMAL);
+         Aggregate      minAggregate           = new Aggregate(fieldName, AggregateOperator.MIN);
+         Aggregate      maxAggregate           = new Aggregate(fieldName, AggregateOperator.MAX);
+         AggregateInput statsAggregateInput    = new AggregateInput();
+         statsAggregateInput.withAggregate(countNonNullAggregate);
+         if(doCountDistinct)
+         {
+            statsAggregateInput.withAggregate(countDistinctAggregate);
+         }
+         if(doSum)
+         {
+            statsAggregateInput.withAggregate(sumAggregate);
+         }
+         if(doAvg)
+         {
+            statsAggregateInput.withAggregate(avgAggregate);
+         }
+         if(doMin)
+         {
+            statsAggregateInput.withAggregate(minAggregate);
+         }
+         if(doMax)
+         {
+            statsAggregateInput.withAggregate(maxAggregate);
+         }
+
+         if(CollectionUtils.nullSafeHasContents(statsAggregateInput.getAggregates()))
+         {
+            statsAggregateInput.setTableName(tableName);
+            filter.setOrderBys(new ArrayList<>());
+            statsAggregateInput.setFilter(filter);
+            AggregateOutput statsAggregateOutput = new AggregateAction().execute(statsAggregateInput);
+            AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0);
+
+            statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate));
+            if(doCountDistinct)
+            {
+               statsRecord.setValue(countDistinctField.getName(), statsAggregateResult.getAggregateValue(countDistinctAggregate));
+            }
+            if(doSum)
+            {
+               statsRecord.setValue(sumField.getName(), statsAggregateResult.getAggregateValue(sumAggregate));
+            }
+            if(doAvg)
+            {
+               statsRecord.setValue(avgField.getName(), statsAggregateResult.getAggregateValue(avgAggregate));
+            }
+            if(doMin)
+            {
+               statsRecord.setValue(minField.getName(), statsAggregateResult.getAggregateValue(minAggregate));
+            }
+            if(doMax)
+            {
+               statsRecord.setValue(maxField.getName(), statsAggregateResult.getAggregateValue(maxAggregate));
+            }
+         }
+
+         QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(null);
+         fields.forEach(qInstanceEnricher::enrichField);
+
+         QValueFormatter.setDisplayValuesInRecord(fields, statsRecord);
+
+         runBackendStepOutput.addValue("statsFields", fields);
+         runBackendStepOutput.addValue("statsRecord", statsRecord);
+      }
+      catch(Exception e)
+      {
+         throw new QException("Error calculating stats", e);
+      }
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -347,11 +347,11 @@ class QInstanceValidatorTest extends BaseTest
    @Test
    public void test_validateProcessStepWithEmptyName()
    {
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).getStepList().get(0).setName(""),
-      // these used to be an assertion failure - but enricher now sets a default name for backend steps w/ a code name //
+         "Missing name for a step");
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
-      assertValidationSuccess((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).getStepList().get(0).setName(""));
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().get(1).setName(null),
-      assertValidationSuccess((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().get(1).setName(null));
+         "Missing name for a step");
    }
 
 

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java

@@ -85,6 +85,11 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
             sql += " ORDER BY " + makeOrderByClause(table, filter.getOrderBys(), joinsContext);
          }
 
+         if(aggregateInput.getLimit() != null)
+         {
+            sql += " LIMIT " + aggregateInput.getLimit();
+         }
+
          // todo sql customization - can edit sql and/or param list
 
          AggregateOutput       rs      = new AggregateOutput();
@@ -114,9 +119,18 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
                      JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(aggregate.getFieldName());
                      QFieldMetaData                        field                    = fieldAndTableNameOrAlias.field();
 
-                     if(field.getType().equals(QFieldType.INTEGER) && aggregate.getOperator().equals(AggregateOperator.AVG))
+                     QFieldType fieldType = aggregate.getFieldType();
+                     if(fieldType == null)
                      {
-                        field = new QFieldMetaData().withType(QFieldType.DECIMAL);
+                        if(field.getType().equals(QFieldType.INTEGER) && (aggregate.getOperator().equals(AggregateOperator.AVG)))
+                        {
+                           fieldType = QFieldType.DECIMAL;
+                        }
+                     }
+
+                     if(fieldType != null)
+                     {
+                        field = new QFieldMetaData().withType(fieldType);
                      }
 
                      Serializable value = getFieldValueFromResultSet(field, resultSet, selectionIndex++);
@@ -155,7 +169,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
       for(Aggregate aggregate : aggregateInput.getAggregates())
       {
          JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(aggregate.getFieldName());
-         rs.add(aggregate.getOperator() + "(" + escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(fieldAndTableNameOrAlias.field())) + ")");
+         rs.add(aggregate.getOperator().getSqlPrefix() + escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(fieldAndTableNameOrAlias.field())) + ")");
       }
       return (rs);
    }