QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

More stats & aggregates

Browse Source
This commit is contained in:
darin.kelkhoff
2023-03-03 09:51:22 -06:00
parent bbde64b02d
commit ad2eff0e73
2 changed files with 179 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

185
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java
View File

@ -33,6 +33,8 @@ import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
@ -42,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOu
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -49,6 +52,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -59,6 +63,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
*******************************************************************************/
public class TableStatsStep implements BackendStep
{
private static final QLogger LOG = QLogger.getLogger(TableStatsStep.class);
/*******************************************************************************
**
@ -70,6 +77,7 @@ public class TableStatsStep implements BackendStep
{
String tableName = runBackendStepInput.getValueString("tableName");
String fieldName = runBackendStepInput.getValueString("fieldName");
String orderBy = runBackendStepInput.getValueString("orderBy");
String filterJSON = runBackendStepInput.getValueString("filterJSON");
/////////////////////////////////////////
@ -95,15 +103,48 @@ public class TableStatsStep implements BackendStep
QTableMetaData table = QContext.getQInstance().getTable(tableName);
QFieldMetaData field = table.getField(fieldName);
Aggregate aggregate = new Aggregate(fieldName, AggregateOperator.COUNT);
////////////////////////////////////////////
// do a count query grouped by this field //
////////////////////////////////////////////
Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT);
GroupBy groupBy = new GroupBy(field.getType(), fieldName);
if(StringUtils.hasContent(orderBy))
{
if(orderBy.equalsIgnoreCase("count.asc"))
{
filter.withOrderBy(new QFilterOrderByAggregate(aggregate, true));
}
else if(orderBy.equalsIgnoreCase("count.desc"))
{
filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false));
}
else if(orderBy.equalsIgnoreCase(fieldName + ".asc"))
{
filter.withOrderBy(new QFilterOrderBy(fieldName, true));
}
else if(orderBy.equalsIgnoreCase(fieldName + ".desc"))
{
filter.withOrderBy(new QFilterOrderBy(fieldName, false));
}
else
{
LOG.info("Unrecognized orderBy: " + orderBy);
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// always add order by to break ties. these will be the default too, if input didn't supply one //
///////////////////////////////////////////////////////////////////////////////////////////////////
filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false));
filter.withOrderBy(new QFilterOrderBy(fieldName));
Integer limit = 1000; // too big?
AggregateInput aggregateInput = new AggregateInput();
aggregateInput.withAggregate(aggregate);
aggregateInput.withGroupBy(groupBy);
aggregateInput.setTableName(tableName);
aggregateInput.setFilter(filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)));
aggregateInput.setFilter(filter);
aggregateInput.setLimit(limit);
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
@ -122,21 +163,141 @@ public class TableStatsStep implements BackendStep
runBackendStepOutput.addValue("valueCounts", valueCounts);
/////////////////////////////////////////////////////
// now do individual statistics as a pseudo-record //
/////////////////////////////////////////////////////
QFieldMetaData countNonNullField = new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS);
QFieldMetaData countDistinctField = new QFieldMetaData("countDistinct", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS);
QFieldMetaData sumField = new QFieldMetaData("sum", QFieldType.DECIMAL).withDisplayFormat(field.getDisplayFormat());
QFieldMetaData avgField = new QFieldMetaData("average", QFieldType.DECIMAL).withDisplayFormat(field.getDisplayFormat());
QFieldMetaData minField = new QFieldMetaData("min", field.getType()).withDisplayFormat(field.getDisplayFormat());
QFieldMetaData maxField = new QFieldMetaData("max", field.getType()).withDisplayFormat(field.getDisplayFormat());
boolean doCountDistinct = true;
boolean doSum = true;
boolean doAvg = true;
boolean doMin = true;
boolean doMax = true;
if(field.getType().isStringLike())
{
doSum = false;
doAvg = false;
}
if(field.getType().equals(QFieldType.BOOLEAN))
{
doSum = false;
doAvg = false;
doMin = false;
doMax = false;
}
if(field.getType().equals(QFieldType.DATE) || field.getType().equals(QFieldType.DATE_TIME))
{
doSum = false;
doAvg = false; // could this be done?
}
if(StringUtils.hasContent(field.getPossibleValueSourceName()))
{
doSum = false;
doAvg = false;
doMin = false;
doMax = false;
}
ArrayList<QFieldMetaData> fields = new ArrayList<>();
fields.add(countNonNullField);
fields.add(countDistinctField);
if(doSum)
{
fields.add(sumField);
}
if(doAvg)
{
fields.add(avgField);
}
if(doMin)
{
fields.add(minField);
}
if(doMax)
{
fields.add(maxField);
}
QRecord statsRecord = new QRecord();
if(valueCounts.size() < limit)
{
runBackendStepOutput.addValue("countDistinct", valueCounts.size());
statsRecord.setValue(countDistinctField.getName(), valueCounts.size());
doCountDistinct = false;
}
else
{
Aggregate countNonNullAggregate = new Aggregate(fieldName, AggregateOperator.COUNT);
Aggregate countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT);
AggregateInput countDistinctAggregateInput = new AggregateInput();
countDistinctAggregateInput.withAggregate(countDistinctAggregate);
countDistinctAggregateInput.setTableName(tableName);
countDistinctAggregateInput.setFilter(filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)));
AggregateOutput countDistinctAggregateOutput = new AggregateAction().execute(countDistinctAggregateInput);
AggregateResult countDistinctAggregateResult = countDistinctAggregateOutput.getResults().get(0);
runBackendStepOutput.addValue("countDistinct", countDistinctAggregateResult.getAggregateValue(countDistinctAggregate));
Aggregate sumAggregate = new Aggregate(fieldName, AggregateOperator.SUM);
Aggregate avgAggregate = new Aggregate(fieldName, AggregateOperator.AVG);
Aggregate minAggregate = new Aggregate(fieldName, AggregateOperator.MIN);
Aggregate maxAggregate = new Aggregate(fieldName, AggregateOperator.MAX);
AggregateInput statsAggregateInput = new AggregateInput();
statsAggregateInput.withAggregate(countNonNullAggregate);
if(doCountDistinct)
{
statsAggregateInput.withAggregate(countDistinctAggregate);
}
if(doSum)
{
statsAggregateInput.withAggregate(sumAggregate);
}
if(doAvg)
{
statsAggregateInput.withAggregate(avgAggregate);
}
if(doMin)
{
statsAggregateInput.withAggregate(minAggregate);
}
if(doMax)
{
statsAggregateInput.withAggregate(maxAggregate);
}
if(CollectionUtils.nullSafeHasContents(statsAggregateInput.getAggregates()))
{
statsAggregateInput.setTableName(tableName);
filter.setOrderBys(new ArrayList<>());
statsAggregateInput.setFilter(filter);
AggregateOutput statsAggregateOutput = new AggregateAction().execute(statsAggregateInput);
AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0);
statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate));
if(doCountDistinct)
{
statsRecord.setValue(countDistinctField.getName(), statsAggregateResult.getAggregateValue(countDistinctAggregate));
}
if(doSum)
{
statsRecord.setValue(sumField.getName(), statsAggregateResult.getAggregateValue(sumAggregate));
}
if(doAvg)
{
statsRecord.setValue(avgField.getName(), statsAggregateResult.getAggregateValue(avgAggregate));
}
if(doMin)
{
statsRecord.setValue(minField.getName(), statsAggregateResult.getAggregateValue(minAggregate));
}
if(doMax)
{
statsRecord.setValue(maxField.getName(), statsAggregateResult.getAggregateValue(maxAggregate));
}
}
QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(null);
fields.forEach(qInstanceEnricher::enrichField);
QValueFormatter.setDisplayValuesInRecord(fields, statsRecord);
runBackendStepOutput.addValue("statsFields", fields);
runBackendStepOutput.addValue("statsRecord", statsRecord);
}
catch(Exception e)
{

7
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java
View File

@ -119,7 +119,12 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(aggregate.getFieldName());
QFieldMetaData field = fieldAndTableNameOrAlias.field();
if(field.getType().equals(QFieldType.INTEGER) && aggregate.getOperator().equals(AggregateOperator.AVG))
if(field.getType().equals(QFieldType.INTEGER) && (aggregate.getOperator().equals(AggregateOperator.AVG) || aggregate.getOperator().equals(AggregateOperator.SUM)))
{
field = new QFieldMetaData().withType(QFieldType.DECIMAL);
}
if(aggregate.getOperator().equals(AggregateOperator.COUNT) || aggregate.getOperator().equals(AggregateOperator.COUNT_DISTINCT))
{
field = new QFieldMetaData().withType(QFieldType.DECIMAL);
}