QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make column stats treat "" and null the same (as a non-value or blank)

Browse Source
This commit is contained in:
darin.kelkhoff
2023-06-01 15:02:32 -05:00
parent 6455171cee
commit f5516c8122
6 changed files with 680 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryAggregateAction.java Normal file
View File

@ -0,0 +1,53 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
/*******************************************************************************
** In-memory version of aggregate action.
**
*******************************************************************************/
public class MemoryAggregateAction implements AggregateInterface
{
/*******************************************************************************
**
*******************************************************************************/
public AggregateOutput execute(AggregateInput aggregateInput) throws QException
{
try
{
return (MemoryRecordStore.getInstance().aggregate(aggregateInput));
}
catch(Exception e)
{
throw new QException("Error executing aggregate", e);
}
}
}

12
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
@ -74,6 +75,17 @@ public class MemoryBackendModule implements QBackendModuleInterface
/*******************************************************************************
**
*******************************************************************************/
@Override
public AggregateInterface getAggregateInterface()
{
return new MemoryAggregateAction();
}
/*******************************************************************************
**
*******************************************************************************/

275
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java
View File

@ -23,22 +23,36 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
import java.io.Serializable;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
@ -525,4 +539,265 @@ public class MemoryRecordStore
return (actionInputs);
}
/*******************************************************************************
**
*******************************************************************************/
public AggregateOutput aggregate(AggregateInput aggregateInput) throws QException
{
//////////////////////
// first do a query //
//////////////////////
QueryInput queryInput = new QueryInput();
queryInput.setTableName(aggregateInput.getTableName());
queryInput.setFilter(aggregateInput.getFilter());
queryInput.setQueryJoins(aggregateInput.getQueryJoins());
List<QRecord> queryResult = query(queryInput);
List<AggregateResult> results = new ArrayList<>();
List<GroupBy> groupBys = CollectionUtils.nonNullList(aggregateInput.getGroupBys());
List<Aggregate> aggregates = CollectionUtils.nonNullList(aggregateInput.getAggregates());
/////////////////////
// do the group-by //
/////////////////////
ListingHash<List<Serializable>, QRecord> bins = new ListingHash<>();
for(QRecord record : queryResult)
{
List<Serializable> groupByValues = new ArrayList<>(groupBys.size());
for(GroupBy groupBy : groupBys)
{
Serializable groupByValue = record.getValue(groupBy.getFieldName());
if(groupBy.getType() != null)
{
groupByValue = ValueUtils.getValueAsFieldType(groupBy.getType(), groupByValue);
}
groupByValues.add(groupByValue);
}
bins.add(groupByValues, record);
}
////////////////////////
// do the aggregating //
////////////////////////
for(Map.Entry<List<Serializable>, List<QRecord>> entry : bins.entrySet())
{
List<Serializable> groupByValueList = entry.getKey();
List<QRecord> records = entry.getValue();
AggregateResult aggregateResult = new AggregateResult();
results.add(aggregateResult);
////////////////////////////////////////////
// set the group-by values in this result //
////////////////////////////////////////////
Map<GroupBy, Serializable> groupByValues = new HashMap<>();
aggregateResult.setGroupByValues(groupByValues);
for(int i = 0; i < groupBys.size(); i++)
{
GroupBy groupBy = groupBys.get(i);
Serializable value = groupByValueList.get(i);
groupByValues.put(groupBy, value);
}
////////////////////////////
// compute the aggregates //
////////////////////////////
Map<Aggregate, Serializable> aggregateValues = new HashMap<>();
aggregateResult.setAggregateValues(aggregateValues);
for(Aggregate aggregate : aggregates)
{
Serializable aggregateValue = computeAggregate(records, aggregate, aggregateInput.getTable());
aggregateValues.put(aggregate, aggregateValue);
}
}
/////////////////////
// sort the result //
/////////////////////
if(aggregateInput.getFilter() != null && CollectionUtils.nullSafeHasContents(aggregateInput.getFilter().getOrderBys()))
{
Comparator<AggregateResult> comparator = null;
Comparator<Serializable> serializableComparator = (Serializable a, Serializable b) ->
{
if(a == null && b == null)
{
return (0);
}
else if(a == null)
{
return (1);
}
else if(b == null)
{
return (-1);
}
return ((Comparable) a).compareTo(b);
};
////////////////////////////////////////////////
// build a comparator out of all the orderBys //
////////////////////////////////////////////////
for(QFilterOrderBy orderBy : aggregateInput.getFilter().getOrderBys())
{
Function<AggregateResult, Serializable> keyExtractor = aggregateResult ->
{
if(orderBy instanceof QFilterOrderByGroupBy orderByGroupBy)
{
return aggregateResult.getGroupByValue(orderByGroupBy.getGroupBy());
}
else if(orderBy instanceof QFilterOrderByAggregate orderByAggregate)
{
return aggregateResult.getAggregateValue(orderByAggregate.getAggregate());
}
else
{
throw (new IllegalStateException("Unexpected orderBy [" + orderBy + "] in aggregate"));
}
};
if(comparator == null)
{
comparator = Comparator.comparing(keyExtractor, serializableComparator);
}
else
{
comparator = comparator.thenComparing(keyExtractor, serializableComparator);
}
if(!orderBy.getIsAscending())
{
comparator = comparator.reversed();
}
}
///////////////////////////////////////
// sort the list with the comparator //
///////////////////////////////////////
results.sort(comparator);
}
AggregateOutput aggregateOutput = new AggregateOutput();
aggregateOutput.setResults(results);
return (aggregateOutput);
}
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("checkstyle:indentation")
private static Serializable computeAggregate(List<QRecord> records, Aggregate aggregate, QTableMetaData table)
{
String fieldName = aggregate.getFieldName();
AggregateOperator operator = aggregate.getOperator();
QFieldType fieldType;
if(aggregate.getFieldType() == null)
{
// todo - joins probably?
QFieldMetaData field = table.getField(fieldName);
if(field.getType().equals(QFieldType.INTEGER) && (operator.equals(AggregateOperator.AVG)))
{
fieldType = QFieldType.DECIMAL;
}
else if(operator.equals(AggregateOperator.COUNT) || operator.equals(AggregateOperator.COUNT_DISTINCT))
{
fieldType = QFieldType.INTEGER;
}
else
{
fieldType = field.getType();
}
}
else
{
fieldType = aggregate.getFieldType();
}
Serializable aggregateValue = switch(operator)
{
case COUNT -> records.stream()
.filter(r -> r.getValue(fieldName) != null)
.count();
case COUNT_DISTINCT -> records.stream()
.filter(r -> r.getValue(fieldName) != null)
.map(r -> r.getValue(fieldName))
.collect(Collectors.toSet())
.size();
case SUM -> switch(fieldType)
{
case INTEGER -> records.stream()
.filter(r -> r.getValue(fieldName) != null)
.mapToInt(r -> r.getValueInteger(fieldName))
.sum();
case DECIMAL -> records.stream()
.filter(r -> r.getValue(fieldName) != null)
.map(r -> r.getValueBigDecimal(fieldName))
.reduce(BigDecimal.ZERO, BigDecimal::add);
default -> throw (new IllegalArgumentException("Cannot perform " + operator + " aggregate on " + fieldType + " field."));
};
case MIN -> switch(fieldType)
{
case INTEGER -> records.stream()
.filter(r -> r.getValue(fieldName) != null)
.mapToInt(r -> r.getValueInteger(fieldName))
.min()
.stream().boxed().findFirst().orElse(null);
case DECIMAL, STRING, DATE, DATE_TIME ->
{
Optional<Serializable> serializable = records.stream()
.filter(r -> r.getValue(fieldName) != null)
.map(r -> ((Comparable) ValueUtils.getValueAsFieldType(fieldType, r.getValue(fieldName))))
.min(Comparator.naturalOrder())
.map(c -> (Serializable) c);
yield serializable.orElse(null);
}
default -> throw (new IllegalArgumentException("Cannot perform " + operator + " aggregate on " + fieldType + " field."));
};
case MAX -> switch(fieldType)
{
case INTEGER -> records.stream()
.filter(r -> r.getValue(fieldName) != null)
.mapToInt(r -> r.getValueInteger(fieldName))
.max()
.stream().boxed().findFirst().orElse(null);
case DECIMAL, STRING, DATE, DATE_TIME ->
{
Optional<Serializable> serializable = records.stream()
.filter(r -> r.getValue(fieldName) != null)
.map(r -> ((Comparable) ValueUtils.getValueAsFieldType(fieldType, r.getValue(fieldName))))
.max(Comparator.naturalOrder())
.map(c -> (Serializable) c);
yield serializable.orElse(null);
}
default -> throw (new IllegalArgumentException("Cannot perform " + operator + " aggregate on " + fieldType + " field."));
};
case AVG -> switch(fieldType)
{
case INTEGER -> records.stream()
.filter(r -> r.getValue(fieldName) != null)
.mapToInt(r -> r.getValueInteger(fieldName))
.average()
.stream().boxed().findFirst().orElse(null);
case DECIMAL -> records.stream()
.filter(r -> r.getValue(fieldName) != null)
.mapToDouble(r -> r.getValueBigDecimal(fieldName).doubleValue())
.average()
.stream().boxed().map(d -> new BigDecimal(d)).findFirst().orElse(null);
default -> throw (new IllegalArgumentException("Cannot perform " + operator + " aggregate on " + fieldType + " field."));
};
};
return ValueUtils.getValueAsFieldType(fieldType, aggregateValue);
}
}

159
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java
View File

@ -24,8 +24,13 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.DateTimeGroupBy;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
@ -45,7 +50,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOu
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
@ -59,6 +64,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@ -140,6 +146,14 @@ public class ColumnStatsStep implements BackendStep
Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
GroupBy groupBy = new GroupBy(field.getType(), fieldName);
// todo - something here about "by-date, not time"
if(field.getType().equals(QFieldType.DATE_TIME))
{
// groupBy = new GroupBy(field.getType(), fieldName, "DATE(%s)");
String sqlExpression = DateTimeGroupBy.HOUR.getSqlExpression();
groupBy = new GroupBy(QFieldType.STRING, fieldName, sqlExpression);
}
if(StringUtils.hasContent(orderBy))
{
if(orderBy.equalsIgnoreCase("count.asc"))
@ -152,11 +166,11 @@ public class ColumnStatsStep implements BackendStep
}
else if(orderBy.equalsIgnoreCase(fieldName + ".asc"))
{
filter.withOrderBy(new QFilterOrderBy(fieldName, true));
filter.withOrderBy(new QFilterOrderByGroupBy(groupBy, true));
}
else if(orderBy.equalsIgnoreCase(fieldName + ".desc"))
{
filter.withOrderBy(new QFilterOrderBy(fieldName, false));
filter.withOrderBy(new QFilterOrderByGroupBy(groupBy, false));
}
else
{
@ -168,7 +182,7 @@ public class ColumnStatsStep implements BackendStep
// always add order by to break ties. these will be the default too, if input didn't supply one //
///////////////////////////////////////////////////////////////////////////////////////////////////
filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false));
filter.withOrderBy(new QFilterOrderBy(fieldName));
filter.withOrderBy(new QFilterOrderByGroupBy(groupBy));
Integer limit = 1000; // too big?
AggregateInput aggregateInput = new AggregateInput();
@ -192,6 +206,14 @@ public class ColumnStatsStep implements BackendStep
Integer count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate));
valueCounts.add(new QRecord().withValue(fieldName, value).withValue("count", count));
}
//////////////////////////////////////////////////////////////////////////////////////////////////
// so... our json serialization causes both "" and null values to go to the frontend as null... //
// so we get 2 rows, but they look the same to the frontend. //
// turns out, users (probably?) don't care about the difference, so let's merge "" and null! //
//////////////////////////////////////////////////////////////////////////////////////////////////
Integer rowsWithAValueToDecrease = mergeEmptyStringAndNull(field, fieldName, valueCounts, orderBy);
QFieldMetaData countField = new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS).withLabel("Count");
QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator();
@ -315,28 +337,49 @@ public class ColumnStatsStep implements BackendStep
statsAggregateInput.withQueryJoin(queryJoin);
}
AggregateOutput statsAggregateOutput = new AggregateAction().execute(statsAggregateInput);
AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0);
if(CollectionUtils.nullSafeHasContents(statsAggregateOutput.getResults()))
{
AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0);
statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate));
if(doCountDistinct)
{
statsRecord.setValue(countDistinctField.getName(), statsAggregateResult.getAggregateValue(countDistinctAggregate));
}
if(doSum)
{
statsRecord.setValue(sumField.getName(), statsAggregateResult.getAggregateValue(sumAggregate));
}
if(doAvg)
{
statsRecord.setValue(avgField.getName(), statsAggregateResult.getAggregateValue(avgAggregate));
}
if(doMin)
{
statsRecord.setValue(minField.getName(), statsAggregateResult.getAggregateValue(minAggregate));
}
if(doMax)
{
statsRecord.setValue(maxField.getName(), statsAggregateResult.getAggregateValue(maxAggregate));
statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate));
if(doCountDistinct)
{
statsRecord.setValue(countDistinctField.getName(), statsAggregateResult.getAggregateValue(countDistinctAggregate));
}
if(doSum)
{
statsRecord.setValue(sumField.getName(), statsAggregateResult.getAggregateValue(sumAggregate));
}
if(doAvg)
{
statsRecord.setValue(avgField.getName(), statsAggregateResult.getAggregateValue(avgAggregate));
}
if(doMin)
{
statsRecord.setValue(minField.getName(), statsAggregateResult.getAggregateValue(minAggregate));
}
if(doMax)
{
statsRecord.setValue(maxField.getName(), statsAggregateResult.getAggregateValue(maxAggregate));
}
if(rowsWithAValueToDecrease != null)
{
///////////////////////////////////////////////////////////////////////////////////////////////
// this is in case we merged any "" and null values - //
// we need to take away however many ""'s there were from countNonNull (treat those as null) //
// and decrease unique values by 1 //
///////////////////////////////////////////////////////////////////////////////////////////////
try
{
statsRecord.setValue(countNonNullField.getName(), statsRecord.getValueInteger(countNonNullField.getName()) - rowsWithAValueToDecrease);
statsRecord.setValue(countDistinctField.getName(), statsRecord.getValueInteger(countDistinctField.getName()) - 1);
}
catch(Exception e)
{
LOG.warn("Error decreasing by non-null empty string count", e, logPair("fieldName", fieldName), logPair("tableName", tableName));
}
}
}
}
@ -354,4 +397,70 @@ public class ColumnStatsStep implements BackendStep
}
}
/*******************************************************************************
**
*******************************************************************************/
private Integer mergeEmptyStringAndNull(QFieldMetaData field, String fieldName, ArrayList<QRecord> valueCounts, String orderBy)
{
if(field.getType().isStringLike())
{
Integer nullCount = null;
Integer emptyStringCount = null;
for(QRecord record : valueCounts)
{
if("".equals(record.getValue(fieldName)))
{
emptyStringCount = record.getValueInteger("count");
}
else if(record.getValue(fieldName) == null)
{
nullCount = record.getValueInteger("count");
}
}
if(nullCount != null && emptyStringCount != null)
{
Iterator<QRecord> iterator = valueCounts.iterator();
while(iterator.hasNext())
{
QRecord record = iterator.next();
if("".equals(record.getValue(fieldName)))
{
iterator.remove();
}
else if(record.getValue(fieldName) == null)
{
record.setValue("count", nullCount + emptyStringCount);
}
}
///////////////////////////////////////////////////
// re-sort the records, as the counts may change //
///////////////////////////////////////////////////
if(StringUtils.hasContent(orderBy))
{
if(orderBy.toLowerCase().startsWith("count."))
{
valueCounts.sort(Comparator.comparing(r -> r.getValueInteger("count")));
}
else
{
valueCounts.sort(Comparator.comparing(r -> Objects.requireNonNullElse(r.getValueString(fieldName), "")));
}
if(orderBy.toLowerCase().endsWith(".desc"))
{
Collections.reverse(valueCounts);
}
}
return (emptyStringCount);
}
}
return (null);
}
}

145
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java
View File

@ -22,12 +22,14 @@
package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.Month;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@ -35,6 +37,14 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
@ -52,6 +62,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
@ -59,6 +70,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ -514,6 +526,139 @@ class MemoryBackendModuleTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testAggregate() throws QException
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(List.of(
new QRecord().withValue("noOfShoes", 1).withValue("lastName", "Simpson"),
new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Simpson"),
new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Flanders"),
new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Flanders"),
new QRecord().withValue("noOfShoes", 3).withValue("lastName", "Flanders"),
new QRecord().withValue("noOfShoes", null).withValue("lastName", "Flanders")
));
new InsertAction().execute(insertInput);
{
////////////////////////////////
// do some integer aggregates //
////////////////////////////////
AggregateInput aggregateInput = new AggregateInput();
aggregateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM));
aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.COUNT));
aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.COUNT_DISTINCT));
aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.MIN));
aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.MAX));
aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.AVG));
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
assertEquals(1, aggregateOutput.getResults().size());
AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
assertEquals(10, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals(5, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.COUNT)));
assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.COUNT_DISTINCT)));
assertEquals(1, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.MIN)));
assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.MAX)));
assertEquals(new BigDecimal(2), aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.AVG)));
}
{
///////////////////////////////
// do some string aggregates //
///////////////////////////////
AggregateInput aggregateInput = new AggregateInput();
aggregateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.COUNT));
aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.COUNT_DISTINCT));
aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.MIN));
aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.MAX));
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
assertEquals(1, aggregateOutput.getResults().size());
AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
assertEquals(6, aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.COUNT)));
assertEquals(2, aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.COUNT_DISTINCT)));
assertEquals("Flanders", aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.MIN)));
assertEquals("Simpson", aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.MAX)));
assertThatThrownBy(() -> new AggregateAction().execute(aggregateInput.withAggregates(List.of(new Aggregate("lastName", AggregateOperator.SUM))))).hasStackTraceContaining("Cannot perform SUM");
assertThatThrownBy(() -> new AggregateAction().execute(aggregateInput.withAggregates(List.of(new Aggregate("lastName", AggregateOperator.AVG))))).hasStackTraceContaining("Cannot perform AVG");
}
{
////////////////////
// do a group-bys //
////////////////////
AggregateInput aggregateInput = new AggregateInput();
aggregateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM));
aggregateInput.withGroupBy(new GroupBy(QFieldType.STRING, "lastName"));
{
aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM))));
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
assertEquals(2, aggregateOutput.getResults().size());
AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
aggregateResult = aggregateOutput.getResults().get(1);
assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
}
{
///////////////////////////////////////////////////////////////////////////
// with all different versions of order-by (agg or groupBy, asc or desc) //
///////////////////////////////////////////////////////////////////////////
aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM)).withIsAscending(false)));
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
assertEquals(2, aggregateOutput.getResults().size());
AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
aggregateResult = aggregateOutput.getResults().get(1);
assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
}
{
aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByGroupBy(new GroupBy(QFieldType.STRING, "lastName"))));
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
assertEquals(2, aggregateOutput.getResults().size());
AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
aggregateResult = aggregateOutput.getResults().get(1);
assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
}
{
aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByGroupBy(new GroupBy(QFieldType.STRING, "lastName")).withIsAscending(false)));
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
assertEquals(2, aggregateOutput.getResults().size());
AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
aggregateResult = aggregateOutput.getResults().get(1);
assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
}
}
}
/*******************************************************************************
**
*******************************************************************************/

61
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java Normal file
View File

@ -0,0 +1,61 @@
package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Unit test for ColumnStatsStep
*******************************************************************************/
class ColumnStatsStepTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testEmptyStringAndNullRollUpTogether() throws QException
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(List.of(
new QRecord().withValue("noOfShoes", 1).withValue("lastName", "Simpson"),
new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Simpson"),
new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Simpson"),
new QRecord().withValue("noOfShoes", 2).withValue("lastName", ""), // this record and the next one -
new QRecord().withValue("noOfShoes", 3).withValue("lastName", null), // this record and the previous - should both come out as null below
new QRecord().withValue("noOfShoes", null).withValue("lastName", "Flanders")
));
new InsertAction().execute(insertInput);
RunBackendStepInput input = new RunBackendStepInput();
input.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
input.addValue("fieldName", "lastName");
input.addValue("orderBy", "count.desc");
RunBackendStepOutput output = new RunBackendStepOutput();
new ColumnStatsStep().run(input, output);
Map<String, Serializable> values = output.getValues();
@SuppressWarnings("unchecked")
List<QRecord> valueCounts = (List<QRecord>) values.get("valueCounts");
assertThat(valueCounts.get(0).getValues()).hasFieldOrPropertyWithValue("lastName", "Simpson").hasFieldOrPropertyWithValue("count", 3);
assertThat(valueCounts.get(1).getValues()).hasFieldOrPropertyWithValue("lastName", null).hasFieldOrPropertyWithValue("count", 2); // here's the assert for the "" and null record above.
assertThat(valueCounts.get(2).getValues()).hasFieldOrPropertyWithValue("lastName", "Flanders").hasFieldOrPropertyWithValue("count", 1);
}
}