Make column stats treat "" and null the same (as a non-value or blank)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryAggregateAction.java

@@ -0,0 +1,53 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
+
+
+import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+
+
+/*******************************************************************************
+ ** In-memory version of aggregate action.
+ **
+ *******************************************************************************/
+public class MemoryAggregateAction implements AggregateInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public AggregateOutput execute(AggregateInput aggregateInput) throws QException
+   {
+      try
+      {
+         return (MemoryRecordStore.getInstance().aggregate(aggregateInput));
+      }
+      catch(Exception e)
+      {
+         throw new QException("Error executing aggregate", e);
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
 
 
+import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
@@ -74,6 +75,17 @@ public class MemoryBackendModule implements QBackendModuleInterface
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public AggregateInterface getAggregateInterface()
+   {
+      return new MemoryAggregateAction();
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java

@@ -23,22 +23,36 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
 
 
 import java.io.Serializable;
+import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
@@ -525,4 +539,265 @@ public class MemoryRecordStore
       return (actionInputs);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public AggregateOutput aggregate(AggregateInput aggregateInput) throws QException
+   {
+      //////////////////////
+      // first do a query //
+      //////////////////////
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(aggregateInput.getTableName());
+      queryInput.setFilter(aggregateInput.getFilter());
+      queryInput.setQueryJoins(aggregateInput.getQueryJoins());
+      List<QRecord> queryResult = query(queryInput);
+
+      List<AggregateResult> results    = new ArrayList<>();
+      List<GroupBy>         groupBys   = CollectionUtils.nonNullList(aggregateInput.getGroupBys());
+      List<Aggregate>       aggregates = CollectionUtils.nonNullList(aggregateInput.getAggregates());
+
+      /////////////////////
+      // do the group-by //
+      /////////////////////
+      ListingHash<List<Serializable>, QRecord> bins = new ListingHash<>();
+      for(QRecord record : queryResult)
+      {
+         List<Serializable> groupByValues = new ArrayList<>(groupBys.size());
+         for(GroupBy groupBy : groupBys)
+         {
+            Serializable groupByValue = record.getValue(groupBy.getFieldName());
+            if(groupBy.getType() != null)
+            {
+               groupByValue = ValueUtils.getValueAsFieldType(groupBy.getType(), groupByValue);
+            }
+            groupByValues.add(groupByValue);
+         }
+
+         bins.add(groupByValues, record);
+      }
+
+      ////////////////////////
+      // do the aggregating //
+      ////////////////////////
+      for(Map.Entry<List<Serializable>, List<QRecord>> entry : bins.entrySet())
+      {
+         List<Serializable> groupByValueList = entry.getKey();
+         List<QRecord>      records          = entry.getValue();
+
+         AggregateResult aggregateResult = new AggregateResult();
+         results.add(aggregateResult);
+
+         ////////////////////////////////////////////
+         // set the group-by values in this result //
+         ////////////////////////////////////////////
+         Map<GroupBy, Serializable> groupByValues = new HashMap<>();
+         aggregateResult.setGroupByValues(groupByValues);
+         for(int i = 0; i < groupBys.size(); i++)
+         {
+            GroupBy      groupBy = groupBys.get(i);
+            Serializable value   = groupByValueList.get(i);
+            groupByValues.put(groupBy, value);
+         }
+
+         ////////////////////////////
+         // compute the aggregates //
+         ////////////////////////////
+         Map<Aggregate, Serializable> aggregateValues = new HashMap<>();
+         aggregateResult.setAggregateValues(aggregateValues);
+
+         for(Aggregate aggregate : aggregates)
+         {
+            Serializable aggregateValue = computeAggregate(records, aggregate, aggregateInput.getTable());
+
+            aggregateValues.put(aggregate, aggregateValue);
+         }
+      }
+
+      /////////////////////
+      // sort the result //
+      /////////////////////
+      if(aggregateInput.getFilter() != null && CollectionUtils.nullSafeHasContents(aggregateInput.getFilter().getOrderBys()))
+      {
+         Comparator<AggregateResult> comparator = null;
+         Comparator<Serializable> serializableComparator = (Serializable a, Serializable b) ->
+         {
+            if(a == null && b == null)
+            {
+               return (0);
+            }
+            else if(a == null)
+            {
+               return (1);
+            }
+            else if(b == null)
+            {
+               return (-1);
+            }
+            return ((Comparable) a).compareTo(b);
+         };
+
+         ////////////////////////////////////////////////
+         // build a comparator out of all the orderBys //
+         ////////////////////////////////////////////////
+         for(QFilterOrderBy orderBy : aggregateInput.getFilter().getOrderBys())
+         {
+            Function<AggregateResult, Serializable> keyExtractor = aggregateResult ->
+            {
+               if(orderBy instanceof QFilterOrderByGroupBy orderByGroupBy)
+               {
+                  return aggregateResult.getGroupByValue(orderByGroupBy.getGroupBy());
+               }
+               else if(orderBy instanceof QFilterOrderByAggregate orderByAggregate)
+               {
+                  return aggregateResult.getAggregateValue(orderByAggregate.getAggregate());
+               }
+               else
+               {
+                  throw (new IllegalStateException("Unexpected orderBy [" + orderBy + "] in aggregate"));
+               }
+            };
+
+            if(comparator == null)
+            {
+               comparator = Comparator.comparing(keyExtractor, serializableComparator);
+            }
+            else
+            {
+               comparator = comparator.thenComparing(keyExtractor, serializableComparator);
+            }
+
+            if(!orderBy.getIsAscending())
+            {
+               comparator = comparator.reversed();
+            }
+         }
+
+         ///////////////////////////////////////
+         // sort the list with the comparator //
+         ///////////////////////////////////////
+         results.sort(comparator);
+      }
+
+      AggregateOutput aggregateOutput = new AggregateOutput();
+      aggregateOutput.setResults(results);
+      return (aggregateOutput);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:indentation")
+   private static Serializable computeAggregate(List<QRecord> records, Aggregate aggregate, QTableMetaData table)
+   {
+      String            fieldName = aggregate.getFieldName();
+      AggregateOperator operator  = aggregate.getOperator();
+      QFieldType        fieldType;
+      if(aggregate.getFieldType() == null)
+      {
+         // todo - joins probably?
+         QFieldMetaData field = table.getField(fieldName);
+         if(field.getType().equals(QFieldType.INTEGER) && (operator.equals(AggregateOperator.AVG)))
+         {
+            fieldType = QFieldType.DECIMAL;
+         }
+         else if(operator.equals(AggregateOperator.COUNT) || operator.equals(AggregateOperator.COUNT_DISTINCT))
+         {
+            fieldType = QFieldType.INTEGER;
+         }
+         else
+         {
+            fieldType = field.getType();
+         }
+      }
+      else
+      {
+         fieldType = aggregate.getFieldType();
+      }
+
+      Serializable aggregateValue = switch(operator)
+      {
+         case COUNT -> records.stream()
+            .filter(r -> r.getValue(fieldName) != null)
+            .count();
+
+         case COUNT_DISTINCT -> records.stream()
+            .filter(r -> r.getValue(fieldName) != null)
+            .map(r -> r.getValue(fieldName))
+            .collect(Collectors.toSet())
+            .size();
+
+         case SUM -> switch(fieldType)
+         {
+            case INTEGER -> records.stream()
+               .filter(r -> r.getValue(fieldName) != null)
+               .mapToInt(r -> r.getValueInteger(fieldName))
+               .sum();
+            case DECIMAL -> records.stream()
+               .filter(r -> r.getValue(fieldName) != null)
+               .map(r -> r.getValueBigDecimal(fieldName))
+               .reduce(BigDecimal.ZERO, BigDecimal::add);
+            default -> throw (new IllegalArgumentException("Cannot perform " + operator + " aggregate on " + fieldType + " field."));
+         };
+
+         case MIN -> switch(fieldType)
+         {
+            case INTEGER -> records.stream()
+               .filter(r -> r.getValue(fieldName) != null)
+               .mapToInt(r -> r.getValueInteger(fieldName))
+               .min()
+               .stream().boxed().findFirst().orElse(null);
+            case DECIMAL, STRING, DATE, DATE_TIME ->
+            {
+               Optional<Serializable> serializable = records.stream()
+                  .filter(r -> r.getValue(fieldName) != null)
+                  .map(r -> ((Comparable) ValueUtils.getValueAsFieldType(fieldType, r.getValue(fieldName))))
+                  .min(Comparator.naturalOrder())
+                  .map(c -> (Serializable) c);
+               yield serializable.orElse(null);
+            }
+            default -> throw (new IllegalArgumentException("Cannot perform " + operator + " aggregate on " + fieldType + " field."));
+         };
+
+         case MAX -> switch(fieldType)
+         {
+            case INTEGER -> records.stream()
+               .filter(r -> r.getValue(fieldName) != null)
+               .mapToInt(r -> r.getValueInteger(fieldName))
+               .max()
+               .stream().boxed().findFirst().orElse(null);
+            case DECIMAL, STRING, DATE, DATE_TIME ->
+            {
+               Optional<Serializable> serializable = records.stream()
+                  .filter(r -> r.getValue(fieldName) != null)
+                  .map(r -> ((Comparable) ValueUtils.getValueAsFieldType(fieldType, r.getValue(fieldName))))
+                  .max(Comparator.naturalOrder())
+                  .map(c -> (Serializable) c);
+               yield serializable.orElse(null);
+            }
+            default -> throw (new IllegalArgumentException("Cannot perform " + operator + " aggregate on " + fieldType + " field."));
+         };
+
+         case AVG -> switch(fieldType)
+         {
+            case INTEGER -> records.stream()
+               .filter(r -> r.getValue(fieldName) != null)
+               .mapToInt(r -> r.getValueInteger(fieldName))
+               .average()
+               .stream().boxed().findFirst().orElse(null);
+            case DECIMAL -> records.stream()
+               .filter(r -> r.getValue(fieldName) != null)
+               .mapToDouble(r -> r.getValueBigDecimal(fieldName).doubleValue())
+               .average()
+               .stream().boxed().map(d -> new BigDecimal(d)).findFirst().orElse(null);
+            default -> throw (new IllegalArgumentException("Cannot perform " + operator + " aggregate on " + fieldType + " field."));
+         };
+      };
+
+      return ValueUtils.getValueAsFieldType(fieldType, aggregateValue);
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java

@@ -24,8 +24,13 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.DateTimeGroupBy;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
 import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
@@ -45,7 +50,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOu
 import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
 import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
@@ -59,6 +64,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -140,6 +146,14 @@ public class ColumnStatsStep implements BackendStep
          Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
          GroupBy   groupBy   = new GroupBy(field.getType(), fieldName);
 
+         // todo - something here about "by-date, not time"
+         if(field.getType().equals(QFieldType.DATE_TIME))
+         {
+            // groupBy = new GroupBy(field.getType(), fieldName, "DATE(%s)");
+            String sqlExpression = DateTimeGroupBy.HOUR.getSqlExpression();
+            groupBy = new GroupBy(QFieldType.STRING, fieldName, sqlExpression);
+         }
+
          if(StringUtils.hasContent(orderBy))
          {
             if(orderBy.equalsIgnoreCase("count.asc"))
@@ -152,11 +166,11 @@ public class ColumnStatsStep implements BackendStep
             }
             else if(orderBy.equalsIgnoreCase(fieldName + ".asc"))
             {
-               filter.withOrderBy(new QFilterOrderBy(fieldName, true));
+               filter.withOrderBy(new QFilterOrderByGroupBy(groupBy, true));
             }
             else if(orderBy.equalsIgnoreCase(fieldName + ".desc"))
             {
-               filter.withOrderBy(new QFilterOrderBy(fieldName, false));
+               filter.withOrderBy(new QFilterOrderByGroupBy(groupBy, false));
             }
             else
             {
@@ -168,7 +182,7 @@ public class ColumnStatsStep implements BackendStep
          // always add order by to break ties.  these will be the default too, if input didn't supply one //
          ///////////////////////////////////////////////////////////////////////////////////////////////////
          filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false));
-         filter.withOrderBy(new QFilterOrderBy(fieldName));
+         filter.withOrderBy(new QFilterOrderByGroupBy(groupBy));
 
          Integer        limit          = 1000; // too big?
          AggregateInput aggregateInput = new AggregateInput();
@@ -192,6 +206,14 @@ public class ColumnStatsStep implements BackendStep
             Integer      count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate));
             valueCounts.add(new QRecord().withValue(fieldName, value).withValue("count", count));
          }
+
+         //////////////////////////////////////////////////////////////////////////////////////////////////
+         // so... our json serialization causes both "" and null values to go to the frontend as null... //
+         // so we get 2 rows, but they look the same to the frontend.                                    //
+         // turns out, users (probably?) don't care about the difference, so let's merge "" and null!    //
+         //////////////////////////////////////////////////////////////////////////////////////////////////
+         Integer rowsWithAValueToDecrease = mergeEmptyStringAndNull(field, fieldName, valueCounts, orderBy);
+
          QFieldMetaData countField = new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS).withLabel("Count");
 
          QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator();
@@ -315,28 +337,49 @@ public class ColumnStatsStep implements BackendStep
                statsAggregateInput.withQueryJoin(queryJoin);
             }
             AggregateOutput statsAggregateOutput = new AggregateAction().execute(statsAggregateInput);
-            AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0);
+            if(CollectionUtils.nullSafeHasContents(statsAggregateOutput.getResults()))
+            {
+               AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0);
 
-            statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate));
+               statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate));
-            if(doCountDistinct)
+               if(doCountDistinct)
-            {
+               {
-               statsRecord.setValue(countDistinctField.getName(), statsAggregateResult.getAggregateValue(countDistinctAggregate));
+                  statsRecord.setValue(countDistinctField.getName(), statsAggregateResult.getAggregateValue(countDistinctAggregate));
-            }
+               }
-            if(doSum)
+               if(doSum)
-            {
+               {
-               statsRecord.setValue(sumField.getName(), statsAggregateResult.getAggregateValue(sumAggregate));
+                  statsRecord.setValue(sumField.getName(), statsAggregateResult.getAggregateValue(sumAggregate));
-            }
+               }
-            if(doAvg)
+               if(doAvg)
-            {
+               {
-               statsRecord.setValue(avgField.getName(), statsAggregateResult.getAggregateValue(avgAggregate));
+                  statsRecord.setValue(avgField.getName(), statsAggregateResult.getAggregateValue(avgAggregate));
-            }
+               }
-            if(doMin)
+               if(doMin)
-            {
+               {
-               statsRecord.setValue(minField.getName(), statsAggregateResult.getAggregateValue(minAggregate));
+                  statsRecord.setValue(minField.getName(), statsAggregateResult.getAggregateValue(minAggregate));
-            }
+               }
-            if(doMax)
+               if(doMax)
-            {
+               {
-               statsRecord.setValue(maxField.getName(), statsAggregateResult.getAggregateValue(maxAggregate));
+                  statsRecord.setValue(maxField.getName(), statsAggregateResult.getAggregateValue(maxAggregate));
+               }
+
+               if(rowsWithAValueToDecrease != null)
+               {
+                  ///////////////////////////////////////////////////////////////////////////////////////////////
+                  // this is in case we merged any "" and null values -                                        //
+                  // we need to take away however many ""'s there were from countNonNull (treat those as null) //
+                  // and decrease unique values by 1                                                           //
+                  ///////////////////////////////////////////////////////////////////////////////////////////////
+                  try
+                  {
+                     statsRecord.setValue(countNonNullField.getName(), statsRecord.getValueInteger(countNonNullField.getName()) - rowsWithAValueToDecrease);
+                     statsRecord.setValue(countDistinctField.getName(), statsRecord.getValueInteger(countDistinctField.getName()) - 1);
+                  }
+                  catch(Exception e)
+                  {
+                     LOG.warn("Error decreasing by non-null empty string count", e, logPair("fieldName", fieldName), logPair("tableName", tableName));
+                  }
+               }
             }
          }
 
@@ -354,4 +397,70 @@ public class ColumnStatsStep implements BackendStep
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Integer mergeEmptyStringAndNull(QFieldMetaData field, String fieldName, ArrayList<QRecord> valueCounts, String orderBy)
+   {
+      if(field.getType().isStringLike())
+      {
+         Integer nullCount        = null;
+         Integer emptyStringCount = null;
+         for(QRecord record : valueCounts)
+         {
+            if("".equals(record.getValue(fieldName)))
+            {
+               emptyStringCount = record.getValueInteger("count");
+            }
+            else if(record.getValue(fieldName) == null)
+            {
+               nullCount = record.getValueInteger("count");
+            }
+         }
+
+         if(nullCount != null && emptyStringCount != null)
+         {
+            Iterator<QRecord> iterator = valueCounts.iterator();
+            while(iterator.hasNext())
+            {
+               QRecord record = iterator.next();
+               if("".equals(record.getValue(fieldName)))
+               {
+                  iterator.remove();
+               }
+               else if(record.getValue(fieldName) == null)
+               {
+                  record.setValue("count", nullCount + emptyStringCount);
+               }
+            }
+
+            ///////////////////////////////////////////////////
+            // re-sort the records, as the counts may change //
+            ///////////////////////////////////////////////////
+            if(StringUtils.hasContent(orderBy))
+            {
+               if(orderBy.toLowerCase().startsWith("count."))
+               {
+                  valueCounts.sort(Comparator.comparing(r -> r.getValueInteger("count")));
+               }
+               else
+               {
+                  valueCounts.sort(Comparator.comparing(r -> Objects.requireNonNullElse(r.getValueString(fieldName), "")));
+               }
+
+               if(orderBy.toLowerCase().endsWith(".desc"))
+               {
+                  Collections.reverse(valueCounts);
+               }
+            }
+
+            return (emptyStringCount);
+         }
+      }
+
+      return (null);
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java

@@ -22,12 +22,14 @@
 package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
 
 
+import java.math.BigDecimal;
 import java.time.LocalDate;
 import java.time.Month;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@@ -35,6 +37,14 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
@@ -52,6 +62,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
@@ -59,6 +70,7 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -514,6 +526,139 @@ class MemoryBackendModuleTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testAggregate() throws QException
+   {
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("noOfShoes", 1).withValue("lastName", "Simpson"),
+         new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Simpson"),
+         new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Flanders"),
+         new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Flanders"),
+         new QRecord().withValue("noOfShoes", 3).withValue("lastName", "Flanders"),
+         new QRecord().withValue("noOfShoes", null).withValue("lastName", "Flanders")
+      ));
+      new InsertAction().execute(insertInput);
+
+      {
+         ////////////////////////////////
+         // do some integer aggregates //
+         ////////////////////////////////
+         AggregateInput aggregateInput = new AggregateInput();
+         aggregateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+         aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM));
+         aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.COUNT));
+         aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.COUNT_DISTINCT));
+         aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.MIN));
+         aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.MAX));
+         aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.AVG));
+         AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+         assertEquals(1, aggregateOutput.getResults().size());
+         AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
+         assertEquals(10, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+         assertEquals(5, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.COUNT)));
+         assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.COUNT_DISTINCT)));
+         assertEquals(1, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.MIN)));
+         assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.MAX)));
+         assertEquals(new BigDecimal(2), aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.AVG)));
+      }
+
+      {
+         ///////////////////////////////
+         // do some string aggregates //
+         ///////////////////////////////
+         AggregateInput aggregateInput = new AggregateInput();
+         aggregateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+         aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.COUNT));
+         aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.COUNT_DISTINCT));
+         aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.MIN));
+         aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.MAX));
+
+         AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+         assertEquals(1, aggregateOutput.getResults().size());
+         AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
+         assertEquals(6, aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.COUNT)));
+         assertEquals(2, aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.COUNT_DISTINCT)));
+         assertEquals("Flanders", aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.MIN)));
+         assertEquals("Simpson", aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.MAX)));
+
+         assertThatThrownBy(() -> new AggregateAction().execute(aggregateInput.withAggregates(List.of(new Aggregate("lastName", AggregateOperator.SUM))))).hasStackTraceContaining("Cannot perform SUM");
+         assertThatThrownBy(() -> new AggregateAction().execute(aggregateInput.withAggregates(List.of(new Aggregate("lastName", AggregateOperator.AVG))))).hasStackTraceContaining("Cannot perform AVG");
+      }
+
+      {
+         ////////////////////
+         // do a group-bys //
+         ////////////////////
+         AggregateInput aggregateInput = new AggregateInput();
+         aggregateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+         aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM));
+         aggregateInput.withGroupBy(new GroupBy(QFieldType.STRING, "lastName"));
+
+         {
+            aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM))));
+
+            AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+            assertEquals(2, aggregateOutput.getResults().size());
+            AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
+            assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+            assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+
+            aggregateResult = aggregateOutput.getResults().get(1);
+            assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+            assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+         }
+         {
+            ///////////////////////////////////////////////////////////////////////////
+            // with all different versions of order-by (agg or groupBy, asc or desc) //
+            ///////////////////////////////////////////////////////////////////////////
+            aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM)).withIsAscending(false)));
+
+            AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+            assertEquals(2, aggregateOutput.getResults().size());
+            AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
+            assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+            assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+
+            aggregateResult = aggregateOutput.getResults().get(1);
+            assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+            assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+         }
+         {
+            aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByGroupBy(new GroupBy(QFieldType.STRING, "lastName"))));
+
+            AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+            assertEquals(2, aggregateOutput.getResults().size());
+            AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
+            assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+            assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+
+            aggregateResult = aggregateOutput.getResults().get(1);
+            assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+            assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+         }
+         {
+            aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByGroupBy(new GroupBy(QFieldType.STRING, "lastName")).withIsAscending(false)));
+
+            AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+            assertEquals(2, aggregateOutput.getResults().size());
+            AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
+            assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+            assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+
+            aggregateResult = aggregateOutput.getResults().get(1);
+            assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+            assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java

@@ -0,0 +1,61 @@
+package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
+
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Unit test for ColumnStatsStep 
+ *******************************************************************************/
+class ColumnStatsStepTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testEmptyStringAndNullRollUpTogether() throws QException
+   {
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("noOfShoes", 1).withValue("lastName", "Simpson"),
+         new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Simpson"),
+         new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Simpson"),
+         new QRecord().withValue("noOfShoes", 2).withValue("lastName", ""), // this record and the next one -
+         new QRecord().withValue("noOfShoes", 3).withValue("lastName", null), // this record and the previous - should both come out as null below
+         new QRecord().withValue("noOfShoes", null).withValue("lastName", "Flanders")
+      ));
+      new InsertAction().execute(insertInput);
+
+      RunBackendStepInput input = new RunBackendStepInput();
+      input.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      input.addValue("fieldName", "lastName");
+      input.addValue("orderBy", "count.desc");
+
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      new ColumnStatsStep().run(input, output);
+
+      Map<String, Serializable> values = output.getValues();
+
+      @SuppressWarnings("unchecked")
+      List<QRecord> valueCounts = (List<QRecord>) values.get("valueCounts");
+
+      assertThat(valueCounts.get(0).getValues()).hasFieldOrPropertyWithValue("lastName", "Simpson").hasFieldOrPropertyWithValue("count", 3);
+      assertThat(valueCounts.get(1).getValues()).hasFieldOrPropertyWithValue("lastName", null).hasFieldOrPropertyWithValue("count", 2); // here's the assert for the "" and null record above.
+      assertThat(valueCounts.get(2).getValues()).hasFieldOrPropertyWithValue("lastName", "Flanders").hasFieldOrPropertyWithValue("count", 1);
+   }
+
+}