diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryAggregateAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryAggregateAction.java
new file mode 100644
index 00000000..bfc2bba0
--- /dev/null
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryAggregateAction.java
@@ -0,0 +1,53 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022. Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
+
+
+import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+
+
+/*******************************************************************************
+ ** In-memory version of aggregate action.
+ **
+ *******************************************************************************/
+public class MemoryAggregateAction implements AggregateInterface
+{
+
+ /*******************************************************************************
+ **
+ *******************************************************************************/
+ public AggregateOutput execute(AggregateInput aggregateInput) throws QException
+ {
+ try
+ {
+ return (MemoryRecordStore.getInstance().aggregate(aggregateInput));
+ }
+ catch(Exception e)
+ {
+ throw new QException("Error executing aggregate", e);
+ }
+ }
+
+}
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java
index cb423dc5..4d6a93cb 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java
@@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
+import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
@@ -74,6 +75,17 @@ public class MemoryBackendModule implements QBackendModuleInterface
+ /*******************************************************************************
+ **
+ *******************************************************************************/
+ @Override
+ public AggregateInterface getAggregateInterface()
+ {
+ return new MemoryAggregateAction();
+ }
+
+
+
/*******************************************************************************
**
*******************************************************************************/
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java
index b25cac78..eeef3387 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java
@@ -23,22 +23,36 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
import java.io.Serializable;
+import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
@@ -525,4 +539,265 @@ public class MemoryRecordStore
return (actionInputs);
}
+
+
+ /*******************************************************************************
+ **
+ *******************************************************************************/
+ public AggregateOutput aggregate(AggregateInput aggregateInput) throws QException
+ {
+ //////////////////////
+ // first do a query //
+ //////////////////////
+ QueryInput queryInput = new QueryInput();
+ queryInput.setTableName(aggregateInput.getTableName());
+ queryInput.setFilter(aggregateInput.getFilter());
+ queryInput.setQueryJoins(aggregateInput.getQueryJoins());
+ List queryResult = query(queryInput);
+
+ List results = new ArrayList<>();
+ List groupBys = CollectionUtils.nonNullList(aggregateInput.getGroupBys());
+ List aggregates = CollectionUtils.nonNullList(aggregateInput.getAggregates());
+
+ /////////////////////
+ // do the group-by //
+ /////////////////////
+ ListingHash, QRecord> bins = new ListingHash<>();
+ for(QRecord record : queryResult)
+ {
+ List groupByValues = new ArrayList<>(groupBys.size());
+ for(GroupBy groupBy : groupBys)
+ {
+ Serializable groupByValue = record.getValue(groupBy.getFieldName());
+ if(groupBy.getType() != null)
+ {
+ groupByValue = ValueUtils.getValueAsFieldType(groupBy.getType(), groupByValue);
+ }
+ groupByValues.add(groupByValue);
+ }
+
+ bins.add(groupByValues, record);
+ }
+
+ ////////////////////////
+ // do the aggregating //
+ ////////////////////////
+ for(Map.Entry, List> entry : bins.entrySet())
+ {
+ List groupByValueList = entry.getKey();
+ List records = entry.getValue();
+
+ AggregateResult aggregateResult = new AggregateResult();
+ results.add(aggregateResult);
+
+ ////////////////////////////////////////////
+ // set the group-by values in this result //
+ ////////////////////////////////////////////
+ Map groupByValues = new HashMap<>();
+ aggregateResult.setGroupByValues(groupByValues);
+ for(int i = 0; i < groupBys.size(); i++)
+ {
+ GroupBy groupBy = groupBys.get(i);
+ Serializable value = groupByValueList.get(i);
+ groupByValues.put(groupBy, value);
+ }
+
+ ////////////////////////////
+ // compute the aggregates //
+ ////////////////////////////
+ Map aggregateValues = new HashMap<>();
+ aggregateResult.setAggregateValues(aggregateValues);
+
+ for(Aggregate aggregate : aggregates)
+ {
+ Serializable aggregateValue = computeAggregate(records, aggregate, aggregateInput.getTable());
+
+ aggregateValues.put(aggregate, aggregateValue);
+ }
+ }
+
+ /////////////////////
+ // sort the result //
+ /////////////////////
+ if(aggregateInput.getFilter() != null && CollectionUtils.nullSafeHasContents(aggregateInput.getFilter().getOrderBys()))
+ {
+ Comparator comparator = null;
+ Comparator serializableComparator = (Serializable a, Serializable b) ->
+ {
+ if(a == null && b == null)
+ {
+ return (0);
+ }
+ else if(a == null)
+ {
+ return (1);
+ }
+ else if(b == null)
+ {
+ return (-1);
+ }
+ return ((Comparable) a).compareTo(b);
+ };
+
+ ////////////////////////////////////////////////
+ // build a comparator out of all the orderBys //
+ ////////////////////////////////////////////////
+ for(QFilterOrderBy orderBy : aggregateInput.getFilter().getOrderBys())
+ {
+ Function keyExtractor = aggregateResult ->
+ {
+ if(orderBy instanceof QFilterOrderByGroupBy orderByGroupBy)
+ {
+ return aggregateResult.getGroupByValue(orderByGroupBy.getGroupBy());
+ }
+ else if(orderBy instanceof QFilterOrderByAggregate orderByAggregate)
+ {
+ return aggregateResult.getAggregateValue(orderByAggregate.getAggregate());
+ }
+ else
+ {
+ throw (new IllegalStateException("Unexpected orderBy [" + orderBy + "] in aggregate"));
+ }
+ };
+
+ if(comparator == null)
+ {
+ comparator = Comparator.comparing(keyExtractor, serializableComparator);
+ }
+ else
+ {
+ comparator = comparator.thenComparing(keyExtractor, serializableComparator);
+ }
+
+ if(!orderBy.getIsAscending())
+ {
+ comparator = comparator.reversed();
+ }
+ }
+
+ ///////////////////////////////////////
+ // sort the list with the comparator //
+ ///////////////////////////////////////
+ results.sort(comparator);
+ }
+
+ AggregateOutput aggregateOutput = new AggregateOutput();
+ aggregateOutput.setResults(results);
+ return (aggregateOutput);
+ }
+
+
+
+ /*******************************************************************************
+ **
+ *******************************************************************************/
+ @SuppressWarnings("checkstyle:indentation")
+ private static Serializable computeAggregate(List records, Aggregate aggregate, QTableMetaData table)
+ {
+ String fieldName = aggregate.getFieldName();
+ AggregateOperator operator = aggregate.getOperator();
+ QFieldType fieldType;
+ if(aggregate.getFieldType() == null)
+ {
+ // todo - joins probably?
+ QFieldMetaData field = table.getField(fieldName);
+ if(field.getType().equals(QFieldType.INTEGER) && (operator.equals(AggregateOperator.AVG)))
+ {
+ fieldType = QFieldType.DECIMAL;
+ }
+ else if(operator.equals(AggregateOperator.COUNT) || operator.equals(AggregateOperator.COUNT_DISTINCT))
+ {
+ fieldType = QFieldType.INTEGER;
+ }
+ else
+ {
+ fieldType = field.getType();
+ }
+ }
+ else
+ {
+ fieldType = aggregate.getFieldType();
+ }
+
+ Serializable aggregateValue = switch(operator)
+ {
+ case COUNT -> records.stream()
+ .filter(r -> r.getValue(fieldName) != null)
+ .count();
+
+ case COUNT_DISTINCT -> records.stream()
+ .filter(r -> r.getValue(fieldName) != null)
+ .map(r -> r.getValue(fieldName))
+ .collect(Collectors.toSet())
+ .size();
+
+ case SUM -> switch(fieldType)
+ {
+ case INTEGER -> records.stream()
+ .filter(r -> r.getValue(fieldName) != null)
+ .mapToInt(r -> r.getValueInteger(fieldName))
+ .sum();
+ case DECIMAL -> records.stream()
+ .filter(r -> r.getValue(fieldName) != null)
+ .map(r -> r.getValueBigDecimal(fieldName))
+ .reduce(BigDecimal.ZERO, BigDecimal::add);
+ default -> throw (new IllegalArgumentException("Cannot perform " + operator + " aggregate on " + fieldType + " field."));
+ };
+
+ case MIN -> switch(fieldType)
+ {
+ case INTEGER -> records.stream()
+ .filter(r -> r.getValue(fieldName) != null)
+ .mapToInt(r -> r.getValueInteger(fieldName))
+ .min()
+ .stream().boxed().findFirst().orElse(null);
+ case DECIMAL, STRING, DATE, DATE_TIME ->
+ {
+ Optional serializable = records.stream()
+ .filter(r -> r.getValue(fieldName) != null)
+ .map(r -> ((Comparable) ValueUtils.getValueAsFieldType(fieldType, r.getValue(fieldName))))
+ .min(Comparator.naturalOrder())
+ .map(c -> (Serializable) c);
+ yield serializable.orElse(null);
+ }
+ default -> throw (new IllegalArgumentException("Cannot perform " + operator + " aggregate on " + fieldType + " field."));
+ };
+
+ case MAX -> switch(fieldType)
+ {
+ case INTEGER -> records.stream()
+ .filter(r -> r.getValue(fieldName) != null)
+ .mapToInt(r -> r.getValueInteger(fieldName))
+ .max()
+ .stream().boxed().findFirst().orElse(null);
+ case DECIMAL, STRING, DATE, DATE_TIME ->
+ {
+ Optional serializable = records.stream()
+ .filter(r -> r.getValue(fieldName) != null)
+ .map(r -> ((Comparable) ValueUtils.getValueAsFieldType(fieldType, r.getValue(fieldName))))
+ .max(Comparator.naturalOrder())
+ .map(c -> (Serializable) c);
+ yield serializable.orElse(null);
+ }
+ default -> throw (new IllegalArgumentException("Cannot perform " + operator + " aggregate on " + fieldType + " field."));
+ };
+
+ case AVG -> switch(fieldType)
+ {
+ case INTEGER -> records.stream()
+ .filter(r -> r.getValue(fieldName) != null)
+ .mapToInt(r -> r.getValueInteger(fieldName))
+ .average()
+ .stream().boxed().findFirst().orElse(null);
+ case DECIMAL -> records.stream()
+ .filter(r -> r.getValue(fieldName) != null)
+ .mapToDouble(r -> r.getValueBigDecimal(fieldName).doubleValue())
+ .average()
+ .stream().boxed().map(d -> new BigDecimal(d)).findFirst().orElse(null);
+ default -> throw (new IllegalArgumentException("Cannot perform " + operator + " aggregate on " + fieldType + " field."));
+ };
+ };
+
+ return ValueUtils.getValueAsFieldType(fieldType, aggregateValue);
+ }
}
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java
index dee5e4d1..2bb90e28 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java
@@ -24,8 +24,13 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.DateTimeGroupBy;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
@@ -45,7 +50,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOu
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
@@ -59,6 +64,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@@ -140,6 +146,14 @@ public class ColumnStatsStep implements BackendStep
Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
GroupBy groupBy = new GroupBy(field.getType(), fieldName);
+ // todo - something here about "by-date, not time"
+ if(field.getType().equals(QFieldType.DATE_TIME))
+ {
+ // groupBy = new GroupBy(field.getType(), fieldName, "DATE(%s)");
+ String sqlExpression = DateTimeGroupBy.HOUR.getSqlExpression();
+ groupBy = new GroupBy(QFieldType.STRING, fieldName, sqlExpression);
+ }
+
if(StringUtils.hasContent(orderBy))
{
if(orderBy.equalsIgnoreCase("count.asc"))
@@ -152,11 +166,11 @@ public class ColumnStatsStep implements BackendStep
}
else if(orderBy.equalsIgnoreCase(fieldName + ".asc"))
{
- filter.withOrderBy(new QFilterOrderBy(fieldName, true));
+ filter.withOrderBy(new QFilterOrderByGroupBy(groupBy, true));
}
else if(orderBy.equalsIgnoreCase(fieldName + ".desc"))
{
- filter.withOrderBy(new QFilterOrderBy(fieldName, false));
+ filter.withOrderBy(new QFilterOrderByGroupBy(groupBy, false));
}
else
{
@@ -168,7 +182,7 @@ public class ColumnStatsStep implements BackendStep
// always add order by to break ties. these will be the default too, if input didn't supply one //
///////////////////////////////////////////////////////////////////////////////////////////////////
filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false));
- filter.withOrderBy(new QFilterOrderBy(fieldName));
+ filter.withOrderBy(new QFilterOrderByGroupBy(groupBy));
Integer limit = 1000; // too big?
AggregateInput aggregateInput = new AggregateInput();
@@ -192,6 +206,14 @@ public class ColumnStatsStep implements BackendStep
Integer count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate));
valueCounts.add(new QRecord().withValue(fieldName, value).withValue("count", count));
}
+
+ //////////////////////////////////////////////////////////////////////////////////////////////////
+ // so... our json serialization causes both "" and null values to go to the frontend as null... //
+ // so we get 2 rows, but they look the same to the frontend. //
+ // turns out, users (probably?) don't care about the difference, so let's merge "" and null! //
+ //////////////////////////////////////////////////////////////////////////////////////////////////
+ Integer rowsWithAValueToDecrease = mergeEmptyStringAndNull(field, fieldName, valueCounts, orderBy);
+
QFieldMetaData countField = new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS).withLabel("Count");
QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator();
@@ -315,28 +337,49 @@ public class ColumnStatsStep implements BackendStep
statsAggregateInput.withQueryJoin(queryJoin);
}
AggregateOutput statsAggregateOutput = new AggregateAction().execute(statsAggregateInput);
- AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0);
+ if(CollectionUtils.nullSafeHasContents(statsAggregateOutput.getResults()))
+ {
+ AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0);
- statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate));
- if(doCountDistinct)
- {
- statsRecord.setValue(countDistinctField.getName(), statsAggregateResult.getAggregateValue(countDistinctAggregate));
- }
- if(doSum)
- {
- statsRecord.setValue(sumField.getName(), statsAggregateResult.getAggregateValue(sumAggregate));
- }
- if(doAvg)
- {
- statsRecord.setValue(avgField.getName(), statsAggregateResult.getAggregateValue(avgAggregate));
- }
- if(doMin)
- {
- statsRecord.setValue(minField.getName(), statsAggregateResult.getAggregateValue(minAggregate));
- }
- if(doMax)
- {
- statsRecord.setValue(maxField.getName(), statsAggregateResult.getAggregateValue(maxAggregate));
+ statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate));
+ if(doCountDistinct)
+ {
+ statsRecord.setValue(countDistinctField.getName(), statsAggregateResult.getAggregateValue(countDistinctAggregate));
+ }
+ if(doSum)
+ {
+ statsRecord.setValue(sumField.getName(), statsAggregateResult.getAggregateValue(sumAggregate));
+ }
+ if(doAvg)
+ {
+ statsRecord.setValue(avgField.getName(), statsAggregateResult.getAggregateValue(avgAggregate));
+ }
+ if(doMin)
+ {
+ statsRecord.setValue(minField.getName(), statsAggregateResult.getAggregateValue(minAggregate));
+ }
+ if(doMax)
+ {
+ statsRecord.setValue(maxField.getName(), statsAggregateResult.getAggregateValue(maxAggregate));
+ }
+
+ if(rowsWithAValueToDecrease != null)
+ {
+ ///////////////////////////////////////////////////////////////////////////////////////////////
+ // this is in case we merged any "" and null values - //
+ // we need to take away however many ""'s there were from countNonNull (treat those as null) //
+ // and decrease unique values by 1 //
+ ///////////////////////////////////////////////////////////////////////////////////////////////
+ try
+ {
+ statsRecord.setValue(countNonNullField.getName(), statsRecord.getValueInteger(countNonNullField.getName()) - rowsWithAValueToDecrease);
+ statsRecord.setValue(countDistinctField.getName(), statsRecord.getValueInteger(countDistinctField.getName()) - 1);
+ }
+ catch(Exception e)
+ {
+ LOG.warn("Error decreasing by non-null empty string count", e, logPair("fieldName", fieldName), logPair("tableName", tableName));
+ }
+ }
}
}
@@ -354,4 +397,70 @@ public class ColumnStatsStep implements BackendStep
}
}
+
+
+ /*******************************************************************************
+ **
+ *******************************************************************************/
+ private Integer mergeEmptyStringAndNull(QFieldMetaData field, String fieldName, ArrayList valueCounts, String orderBy)
+ {
+ if(field.getType().isStringLike())
+ {
+ Integer nullCount = null;
+ Integer emptyStringCount = null;
+ for(QRecord record : valueCounts)
+ {
+ if("".equals(record.getValue(fieldName)))
+ {
+ emptyStringCount = record.getValueInteger("count");
+ }
+ else if(record.getValue(fieldName) == null)
+ {
+ nullCount = record.getValueInteger("count");
+ }
+ }
+
+ if(nullCount != null && emptyStringCount != null)
+ {
+ Iterator iterator = valueCounts.iterator();
+ while(iterator.hasNext())
+ {
+ QRecord record = iterator.next();
+ if("".equals(record.getValue(fieldName)))
+ {
+ iterator.remove();
+ }
+ else if(record.getValue(fieldName) == null)
+ {
+ record.setValue("count", nullCount + emptyStringCount);
+ }
+ }
+
+ ///////////////////////////////////////////////////
+ // re-sort the records, as the counts may change //
+ ///////////////////////////////////////////////////
+ if(StringUtils.hasContent(orderBy))
+ {
+ if(orderBy.toLowerCase().startsWith("count."))
+ {
+ valueCounts.sort(Comparator.comparing(r -> r.getValueInteger("count")));
+ }
+ else
+ {
+ valueCounts.sort(Comparator.comparing(r -> Objects.requireNonNullElse(r.getValueString(fieldName), "")));
+ }
+
+ if(orderBy.toLowerCase().endsWith(".desc"))
+ {
+ Collections.reverse(valueCounts);
+ }
+ }
+
+ return (emptyStringCount);
+ }
+ }
+
+ return (null);
+ }
+
}
diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java
index 23b8ab93..e533691e 100644
--- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java
+++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java
@@ -22,12 +22,14 @@
package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
+import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.Month;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@@ -35,6 +37,14 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
@@ -52,6 +62,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
@@ -59,6 +70,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -514,6 +526,139 @@ class MemoryBackendModuleTest extends BaseTest
+ /*******************************************************************************
+ **
+ *******************************************************************************/
+ @Test
+ void testAggregate() throws QException
+ {
+ InsertInput insertInput = new InsertInput();
+ insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+ insertInput.setRecords(List.of(
+ new QRecord().withValue("noOfShoes", 1).withValue("lastName", "Simpson"),
+ new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Simpson"),
+ new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Flanders"),
+ new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Flanders"),
+ new QRecord().withValue("noOfShoes", 3).withValue("lastName", "Flanders"),
+ new QRecord().withValue("noOfShoes", null).withValue("lastName", "Flanders")
+ ));
+ new InsertAction().execute(insertInput);
+
+ {
+ ////////////////////////////////
+ // do some integer aggregates //
+ ////////////////////////////////
+ AggregateInput aggregateInput = new AggregateInput();
+ aggregateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+ aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM));
+ aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.COUNT));
+ aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.COUNT_DISTINCT));
+ aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.MIN));
+ aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.MAX));
+ aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.AVG));
+ AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+ assertEquals(1, aggregateOutput.getResults().size());
+ AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
+ assertEquals(10, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+ assertEquals(5, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.COUNT)));
+ assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.COUNT_DISTINCT)));
+ assertEquals(1, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.MIN)));
+ assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.MAX)));
+ assertEquals(new BigDecimal(2), aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.AVG)));
+ }
+
+ {
+ ///////////////////////////////
+ // do some string aggregates //
+ ///////////////////////////////
+ AggregateInput aggregateInput = new AggregateInput();
+ aggregateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+ aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.COUNT));
+ aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.COUNT_DISTINCT));
+ aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.MIN));
+ aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.MAX));
+
+ AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+ assertEquals(1, aggregateOutput.getResults().size());
+ AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
+ assertEquals(6, aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.COUNT)));
+ assertEquals(2, aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.COUNT_DISTINCT)));
+ assertEquals("Flanders", aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.MIN)));
+ assertEquals("Simpson", aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.MAX)));
+
+ assertThatThrownBy(() -> new AggregateAction().execute(aggregateInput.withAggregates(List.of(new Aggregate("lastName", AggregateOperator.SUM))))).hasStackTraceContaining("Cannot perform SUM");
+ assertThatThrownBy(() -> new AggregateAction().execute(aggregateInput.withAggregates(List.of(new Aggregate("lastName", AggregateOperator.AVG))))).hasStackTraceContaining("Cannot perform AVG");
+ }
+
+ {
+ ////////////////////
+ // do a group-bys //
+ ////////////////////
+ AggregateInput aggregateInput = new AggregateInput();
+ aggregateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+ aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM));
+ aggregateInput.withGroupBy(new GroupBy(QFieldType.STRING, "lastName"));
+
+ {
+ aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM))));
+
+ AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+ assertEquals(2, aggregateOutput.getResults().size());
+ AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
+ assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+ assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+
+ aggregateResult = aggregateOutput.getResults().get(1);
+ assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+ assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+ }
+ {
+ ///////////////////////////////////////////////////////////////////////////
+ // with all different versions of order-by (agg or groupBy, asc or desc) //
+ ///////////////////////////////////////////////////////////////////////////
+ aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM)).withIsAscending(false)));
+
+ AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+ assertEquals(2, aggregateOutput.getResults().size());
+ AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
+ assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+ assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+
+ aggregateResult = aggregateOutput.getResults().get(1);
+ assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+ assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+ }
+ {
+ aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByGroupBy(new GroupBy(QFieldType.STRING, "lastName"))));
+
+ AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+ assertEquals(2, aggregateOutput.getResults().size());
+ AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
+ assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+ assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+
+ aggregateResult = aggregateOutput.getResults().get(1);
+ assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+ assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+ }
+ {
+ aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByGroupBy(new GroupBy(QFieldType.STRING, "lastName")).withIsAscending(false)));
+
+ AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+ assertEquals(2, aggregateOutput.getResults().size());
+ AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
+ assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+ assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+
+ aggregateResult = aggregateOutput.getResults().get(1);
+ assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
+ assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
+ }
+ }
+ }
+
+
+
/*******************************************************************************
**
*******************************************************************************/
diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java
new file mode 100644
index 00000000..0e7f9ed9
--- /dev/null
+++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java
@@ -0,0 +1,61 @@
+package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
+
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Unit test for ColumnStatsStep
+ *******************************************************************************/
+class ColumnStatsStepTest extends BaseTest
+{
+
+ /*******************************************************************************
+ **
+ *******************************************************************************/
+ @Test
+ void testEmptyStringAndNullRollUpTogether() throws QException
+ {
+ InsertInput insertInput = new InsertInput();
+ insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+ insertInput.setRecords(List.of(
+ new QRecord().withValue("noOfShoes", 1).withValue("lastName", "Simpson"),
+ new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Simpson"),
+ new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Simpson"),
+ new QRecord().withValue("noOfShoes", 2).withValue("lastName", ""), // this record and the next one -
+ new QRecord().withValue("noOfShoes", 3).withValue("lastName", null), // this record and the previous - should both come out as null below
+ new QRecord().withValue("noOfShoes", null).withValue("lastName", "Flanders")
+ ));
+ new InsertAction().execute(insertInput);
+
+ RunBackendStepInput input = new RunBackendStepInput();
+ input.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
+ input.addValue("fieldName", "lastName");
+ input.addValue("orderBy", "count.desc");
+
+ RunBackendStepOutput output = new RunBackendStepOutput();
+ new ColumnStatsStep().run(input, output);
+
+ Map values = output.getValues();
+
+ @SuppressWarnings("unchecked")
+ List valueCounts = (List) values.get("valueCounts");
+
+ assertThat(valueCounts.get(0).getValues()).hasFieldOrPropertyWithValue("lastName", "Simpson").hasFieldOrPropertyWithValue("count", 3);
+ assertThat(valueCounts.get(1).getValues()).hasFieldOrPropertyWithValue("lastName", null).hasFieldOrPropertyWithValue("count", 2); // here's the assert for the "" and null record above.
+ assertThat(valueCounts.get(2).getValues()).hasFieldOrPropertyWithValue("lastName", "Flanders").hasFieldOrPropertyWithValue("count", 1);
+ }
+
+}
\ No newline at end of file