QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13 from Kingsrook/feature/column-stats

Browse Source 
Feature/column stats
This commit is contained in:
tim-chamberlain
2023-03-16 12:07:31 -05:00
committed by GitHub
parent 03e9f27866 b16eaca394
commit c9e86896e5
12 changed files with 509 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java
View File

@ -436,7 +436,7 @@ public class PermissionsHelper
}
}
if(hasPermission(actionInput.getSession(), permissionBaseName, effectivePermissionSubType))
if(hasPermission(QContext.getQSession(), permissionBaseName, effectivePermissionSubType))
{
return (PermissionCheckResult.ALLOW);
}
@ -534,9 +534,9 @@ public class PermissionsHelper
return;
}
if(!hasPermission(actionInput.getSession(), permissionBaseName, effectivePermissionSubType))
if(!hasPermission(QContext.getQSession(), permissionBaseName, effectivePermissionSubType))
{
// LOG.debug("Throwing permission denied for: " + getPermissionName(permissionBaseName, effectivePermissionSubType) + " for " + actionInput.getSession().getUser());
// LOG.debug("Throwing permission denied for: " + getPermissionName(permissionBaseName, effectivePermissionSubType) + " for " + QContext.getQSession().getUser());
throw (new QPermissionDeniedException("Permission denied."));
}
}

34
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java
View File

@ -34,6 +34,7 @@ import java.util.Objects;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QValueException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
@ -67,28 +68,31 @@ public class QPossibleValueTranslator
{
private static final QLogger LOG = QLogger.getLogger(QPossibleValueTranslator.class);
private final QInstance qInstance;
private final QSession session;
///////////////////////////////////////////////////////
// top-level keys are pvsNames (not table names) //
// 2nd-level keys are pkey values from the PVS table //
///////////////////////////////////////////////////////
private Map<String, Map<Serializable, String>> possibleValueCache;
private Map<String, Map<Serializable, String>> possibleValueCache = new HashMap<>();
// todo not commit - remove instance & session - use Context
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public QPossibleValueTranslator()
{
}
/*******************************************************************************
**
*******************************************************************************/
public QPossibleValueTranslator(QInstance qInstance, QSession session)
{
this.qInstance = qInstance;
this.session = session;
this.possibleValueCache = new HashMap<>();
}
@ -141,7 +145,7 @@ public class QPossibleValueTranslator
{
try
{
QTableMetaData joinTable = qInstance.getTable(queryJoin.getJoinTable());
QTableMetaData joinTable = QContext.getQInstance().getTable(queryJoin.getJoinTable());
for(QFieldMetaData field : joinTable.getFields().values())
{
String joinFieldName = Objects.requireNonNullElse(queryJoin.getAlias(), joinTable.getName()) + "." + field.getName();
@ -152,7 +156,7 @@ public class QPossibleValueTranslator
///////////////////////////////////////////////
// avoid circling-back upon the source table //
///////////////////////////////////////////////
QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(field.getPossibleValueSourceName());
QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName());
if(QPossibleValueSourceType.TABLE.equals(possibleValueSource.getType()) && table.getName().equals(possibleValueSource.getTableName()))
{
continue;
@ -212,7 +216,7 @@ public class QPossibleValueTranslator
*******************************************************************************/
public String translatePossibleValue(QFieldMetaData field, Serializable value)
{
QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(field.getPossibleValueSourceName());
QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName());
if(possibleValueSource == null)
{
LOG.error("Missing possible value source named [" + field.getPossibleValueSourceName() + "] when formatting value for field [" + field.getName() + "]");
@ -414,7 +418,7 @@ public class QPossibleValueTranslator
if(queryJoin.getSelect())
{
String aliasOrTableName = Objects.requireNonNullElse(queryJoin.getAlias(), queryJoin.getJoinTable());
primePvsCacheTableListingHashLoader(qInstance.getTable(queryJoin.getJoinTable()), fieldsByPvsTable, pvsesByTable, aliasOrTableName + ".", queryJoin.getJoinTable(), limitedToFieldNames);
primePvsCacheTableListingHashLoader(QContext.getQInstance().getTable(queryJoin.getJoinTable()), fieldsByPvsTable, pvsesByTable, aliasOrTableName + ".", queryJoin.getJoinTable(), limitedToFieldNames);
}
}
@ -474,7 +478,7 @@ public class QPossibleValueTranslator
{
for(QFieldMetaData field : table.getFields().values())
{
QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(field.getPossibleValueSourceName());
QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName());
if(possibleValueSource != null && possibleValueSource.getType().equals(QPossibleValueSourceType.TABLE))
{
if(limitedToFieldNames != null && !limitedToFieldNames.contains(fieldNamePrefix + field.getName()))
@ -510,7 +514,7 @@ public class QPossibleValueTranslator
try
{
String primaryKeyField = qInstance.getTable(tableName).getPrimaryKeyField();
String primaryKeyField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField();
for(List<Serializable> page : CollectionUtils.getPages(values, 1000))
{
@ -531,7 +535,7 @@ public class QPossibleValueTranslator
{
if(possibleValueSource.getType().equals(QPossibleValueSourceType.TABLE))
{
QTableMetaData table = qInstance.getTable(possibleValueSource.getTableName());
QTableMetaData table = QContext.getQInstance().getTable(possibleValueSource.getTableName());
for(String recordLabelField : CollectionUtils.nonNullList(table.getRecordLabelFields()))
{
QFieldMetaData field = table.getField(recordLabelField);

21
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java
View File

@ -282,7 +282,8 @@ public class QValueFormatter
/*******************************************************************************
** For a list of records, set their recordLabels and display values
** For a list of records, set their recordLabels and display values - including
** record label (e.g., from the table meta data).
*******************************************************************************/
public static void setDisplayValuesInRecords(QTableMetaData table, List<QRecord> records)
{
@ -300,6 +301,24 @@ public class QValueFormatter
/*******************************************************************************
** For a list of records, set their recordLabels and display values
*******************************************************************************/
public static void setDisplayValuesInRecords(Collection<QFieldMetaData> fields, List<QRecord> records)
{
if(records == null)
{
return;
}
for(QRecord record : records)
{
setDisplayValuesInRecord(fields, record);
}
}
/*******************************************************************************
** For a list of records, set their display values
*******************************************************************************/

9
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
View File

@ -51,7 +51,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
@ -293,14 +292,6 @@ public class QInstanceEnricher
*******************************************************************************/
private void enrichStep(QStepMetaData step)
{
if(!StringUtils.hasContent(step.getName()) && step instanceof QBackendStepMetaData backendStep)
{
if(backendStep.getCode() != null && backendStep.getCode().getName() != null)
{
step.setName(backendStep.getCode().getName());
}
}
if(!StringUtils.hasContent(step.getLabel()))
{
step.setLabel(nameToLabel(step.getName()));

42
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/Aggregate.java
View File

@ -24,15 +24,18 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.aggregate;
import java.io.Serializable;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
/*******************************************************************************
**
** Define an "aggregate", e.g., to be selected in an Aggregate action.
** Such as SUM(cost).
*******************************************************************************/
public class Aggregate implements Serializable
{
private String fieldName;
private AggregateOperator operator;
private QFieldType fieldType;
@ -55,12 +58,14 @@ public class Aggregate implements Serializable
{
return true;
}
if(o == null || getClass() != o.getClass())
{
return false;
}
Aggregate aggregate = (Aggregate) o;
return Objects.equals(fieldName, aggregate.fieldName) && operator == aggregate.operator;
return Objects.equals(fieldName, aggregate.fieldName) && operator == aggregate.operator && fieldType == aggregate.fieldType;
}
@ -71,7 +76,7 @@ public class Aggregate implements Serializable
@Override
public int hashCode()
{
return Objects.hash(fieldName, operator);
return Objects.hash(fieldName, operator, fieldType);
}
@ -153,4 +158,35 @@ public class Aggregate implements Serializable
return (this);
}
/*******************************************************************************
** Getter for fieldType
*******************************************************************************/
public QFieldType getFieldType()
{
return (this.fieldType);
}
/*******************************************************************************
** Setter for fieldType
*******************************************************************************/
public void setFieldType(QFieldType fieldType)
{
this.fieldType = fieldType;
}
/*******************************************************************************
** Fluent setter for fieldType
*******************************************************************************/
public Aggregate withFieldType(QFieldType fieldType)
{
this.fieldType = fieldType;
return (this);
}
}

35
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateInput.java
View File

@ -38,6 +38,7 @@ public class AggregateInput extends AbstractTableActionInput
private QQueryFilter filter;
private List<Aggregate> aggregates;
private List<GroupBy> groupBys = new ArrayList<>();
private Integer limit;
private List<QueryJoin> queryJoins = null;
@ -234,4 +235,38 @@ public class AggregateInput extends AbstractTableActionInput
return (this);
}
/*******************************************************************************
** Getter for limit
**
*******************************************************************************/
public Integer getLimit()
{
return limit;
}
/*******************************************************************************
** Setter for limit
**
*******************************************************************************/
public void setLimit(Integer limit)
{
this.limit = limit;
}
/*******************************************************************************
** Fluent setter for limit
**
*******************************************************************************/
public AggregateInput withLimit(Integer limit)
{
this.limit = limit;
return (this);
}
}

34
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateOperator.java
View File

@ -27,9 +27,33 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.aggregate;
*******************************************************************************/
public enum AggregateOperator
{
COUNT,
SUM,
MIN,
MAX,
AVG
COUNT("COUNT("),
COUNT_DISTINCT("COUNT(DISTINCT "),
SUM("SUM("),
MIN("MIN("),
MAX("MAX("),
AVG("AVG(");
private final String sqlPrefix;
/*******************************************************************************
**
*******************************************************************************/
AggregateOperator(String sqlPrefix)
{
this.sqlPrefix = sqlPrefix;
}
/*******************************************************************************
** Getter for sqlPrefix
**
*******************************************************************************/
public String getSqlPrefix()
{
return sqlPrefix;
}
}

11
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java
View File

@ -38,6 +38,17 @@ public class GroupBy implements Serializable
/*******************************************************************************
**
*******************************************************************************/
public GroupBy(QFieldType type, String fieldName)
{
this.type = type;
this.fieldName = fieldName;
}
/*******************************************************************************
**
*******************************************************************************/

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java
View File

@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
@ -234,6 +235,12 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
{
this.steps = new HashMap<>();
}
if(!StringUtils.hasContent(step.getName()))
{
throw (new IllegalArgumentException("Attempt to add a process step without a name"));
}
this.steps.put(step.getName(), step);
return (this);
@ -251,6 +258,12 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
{
this.steps = new HashMap<>();
}
if(!StringUtils.hasContent(step.getName()))
{
throw (new IllegalArgumentException("Attempt to add a process step without a name"));
}
this.steps.put(step.getName(), step);
return (this);

318
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java Normal file
View File

@ -0,0 +1,318 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
** This is a single-step process used to provide Column Statistics. These include
** counts per-value for a field, plus things like total count, min, max, avg, based
** on the field type.
*******************************************************************************/
public class ColumnStatsStep implements BackendStep
{
private static final QLogger LOG = QLogger.getLogger(ColumnStatsStep.class);
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
try
{
String tableName = runBackendStepInput.getValueString("tableName");
String fieldName = runBackendStepInput.getValueString("fieldName");
String orderBy = runBackendStepInput.getValueString("orderBy");
String filterJSON = runBackendStepInput.getValueString("filterJSON");
/////////////////////////////////////////
// make sure user may query this table //
/////////////////////////////////////////
PermissionsHelper.checkTablePermissionThrowing(new QueryInput().withTableName(tableName), TablePermissionSubType.READ);
QQueryFilter filter = null;
if(StringUtils.hasContent(filterJSON))
{
filter = JsonUtils.toObject(filterJSON, QQueryFilter.class);
///////////////////////////////////////////////////////////////
// ... remove any order-by that may have been in that filter //
///////////////////////////////////////////////////////////////
filter.setOrderBys(new ArrayList<>());
}
else
{
filter = new QQueryFilter();
}
QTableMetaData table = QContext.getQInstance().getTable(tableName);
QFieldMetaData field = table.getField(fieldName);
////////////////////////////////////////////
// do a count query grouped by this field //
////////////////////////////////////////////
Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
GroupBy groupBy = new GroupBy(field.getType(), fieldName);
if(StringUtils.hasContent(orderBy))
{
if(orderBy.equalsIgnoreCase("count.asc"))
{
filter.withOrderBy(new QFilterOrderByAggregate(aggregate, true));
}
else if(orderBy.equalsIgnoreCase("count.desc"))
{
filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false));
}
else if(orderBy.equalsIgnoreCase(fieldName + ".asc"))
{
filter.withOrderBy(new QFilterOrderBy(fieldName, true));
}
else if(orderBy.equalsIgnoreCase(fieldName + ".desc"))
{
filter.withOrderBy(new QFilterOrderBy(fieldName, false));
}
else
{
LOG.info("Unrecognized orderBy: " + orderBy);
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// always add order by to break ties. these will be the default too, if input didn't supply one //
///////////////////////////////////////////////////////////////////////////////////////////////////
filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false));
filter.withOrderBy(new QFilterOrderBy(fieldName));
Integer limit = 1000; // too big?
AggregateInput aggregateInput = new AggregateInput();
aggregateInput.withAggregate(aggregate);
aggregateInput.withGroupBy(groupBy);
aggregateInput.setTableName(tableName);
aggregateInput.setFilter(filter);
aggregateInput.setLimit(limit);
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
ArrayList<QRecord> valueCounts = new ArrayList<>();
for(AggregateResult result : aggregateOutput.getResults())
{
Serializable value = result.getGroupByValue(groupBy);
Integer count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate));
valueCounts.add(new QRecord().withValue(fieldName, value).withValue("count", count));
}
QFieldMetaData countField = new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS).withLabel("Count");
QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator();
qPossibleValueTranslator.translatePossibleValuesInRecords(table, valueCounts, null, null);
QValueFormatter.setDisplayValuesInRecords(List.of(table.getField(fieldName), countField), valueCounts);
runBackendStepOutput.addValue("valueCounts", valueCounts);
/////////////////////////////////////////////////////
// now do individual statistics as a pseudo-record //
/////////////////////////////////////////////////////
QFieldMetaData countNonNullField = new QFieldMetaData("count", QFieldType.INTEGER).withLabel("Rows with a value").withDisplayFormat(DisplayFormat.COMMAS);
QFieldMetaData countDistinctField = new QFieldMetaData("countDistinct", QFieldType.INTEGER).withLabel("Distinct values").withDisplayFormat(DisplayFormat.COMMAS);
QFieldMetaData sumField = new QFieldMetaData("sum", QFieldType.DECIMAL).withDisplayFormat(field.getDisplayFormat());
QFieldMetaData avgField = new QFieldMetaData("average", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.DECIMAL2_COMMAS);
QFieldMetaData minField = new QFieldMetaData("min", field.getType()).withDisplayFormat(field.getDisplayFormat());
QFieldMetaData maxField = new QFieldMetaData("max", field.getType()).withDisplayFormat(field.getDisplayFormat());
boolean doCountDistinct = true;
boolean doSum = true;
boolean doAvg = true;
boolean doMin = true;
boolean doMax = true;
if(field.getType().isStringLike())
{
doSum = false;
doAvg = false;
}
if(field.getType().equals(QFieldType.BOOLEAN))
{
doSum = false;
doAvg = false;
doMin = false;
doMax = false;
}
if(field.getType().equals(QFieldType.DATE) || field.getType().equals(QFieldType.DATE_TIME))
{
doSum = false;
doAvg = false; // could this be done?
}
if(StringUtils.hasContent(field.getPossibleValueSourceName()))
{
doSum = false;
doAvg = false;
doMin = false;
doMax = false;
}
if(field.getName().equals(table.getPrimaryKeyField()))
{
doSum = false;
}
ArrayList<QFieldMetaData> fields = new ArrayList<>();
fields.add(countNonNullField);
fields.add(countDistinctField);
if(doSum)
{
fields.add(sumField);
}
if(doAvg)
{
fields.add(avgField);
}
if(doMin)
{
fields.add(minField);
}
if(doMax)
{
fields.add(maxField);
}
QRecord statsRecord = new QRecord();
if(valueCounts.size() < limit)
{
statsRecord.setValue(countDistinctField.getName(), valueCounts.size());
doCountDistinct = false;
}
/////////////////////////////////////////////////////////////////////////////////
// just in case any of these don't fit in an integer, use decimal for them all //
/////////////////////////////////////////////////////////////////////////////////
Aggregate countNonNullAggregate = new Aggregate(fieldName, AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
Aggregate countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT).withFieldType(QFieldType.DECIMAL);
Aggregate sumAggregate = new Aggregate(fieldName, AggregateOperator.SUM).withFieldType(QFieldType.DECIMAL);
Aggregate avgAggregate = new Aggregate(fieldName, AggregateOperator.AVG).withFieldType(QFieldType.DECIMAL);
Aggregate minAggregate = new Aggregate(fieldName, AggregateOperator.MIN);
Aggregate maxAggregate = new Aggregate(fieldName, AggregateOperator.MAX);
AggregateInput statsAggregateInput = new AggregateInput();
statsAggregateInput.withAggregate(countNonNullAggregate);
if(doCountDistinct)
{
statsAggregateInput.withAggregate(countDistinctAggregate);
}
if(doSum)
{
statsAggregateInput.withAggregate(sumAggregate);
}
if(doAvg)
{
statsAggregateInput.withAggregate(avgAggregate);
}
if(doMin)
{
statsAggregateInput.withAggregate(minAggregate);
}
if(doMax)
{
statsAggregateInput.withAggregate(maxAggregate);
}
if(CollectionUtils.nullSafeHasContents(statsAggregateInput.getAggregates()))
{
statsAggregateInput.setTableName(tableName);
filter.setOrderBys(new ArrayList<>());
statsAggregateInput.setFilter(filter);
AggregateOutput statsAggregateOutput = new AggregateAction().execute(statsAggregateInput);
AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0);
statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate));
if(doCountDistinct)
{
statsRecord.setValue(countDistinctField.getName(), statsAggregateResult.getAggregateValue(countDistinctAggregate));
}
if(doSum)
{
statsRecord.setValue(sumField.getName(), statsAggregateResult.getAggregateValue(sumAggregate));
}
if(doAvg)
{
statsRecord.setValue(avgField.getName(), statsAggregateResult.getAggregateValue(avgAggregate));
}
if(doMin)
{
statsRecord.setValue(minField.getName(), statsAggregateResult.getAggregateValue(minAggregate));
}
if(doMax)
{
statsRecord.setValue(maxField.getName(), statsAggregateResult.getAggregateValue(maxAggregate));
}
}
QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(null);
fields.forEach(qInstanceEnricher::enrichField);
QValueFormatter.setDisplayValuesInRecord(fields, statsRecord);
runBackendStepOutput.addValue("statsFields", fields);
runBackendStepOutput.addValue("statsRecord", statsRecord);
}
catch(Exception e)
{
throw new QException("Error calculating stats", e);
}
}
}

10
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java
View File

@ -347,11 +347,11 @@ class QInstanceValidatorTest extends BaseTest
@Test
public void test_validateProcessStepWithEmptyName()
{
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// these used to be an assertion failure - but enricher now sets a default name for backend steps w/ a code name //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
assertValidationSuccess((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).getStepList().get(0).setName(""));
assertValidationSuccess((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().get(1).setName(null));
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).getStepList().get(0).setName(""),
"Missing name for a step");
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().get(1).setName(null),
"Missing name for a step");
}

20
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java
View File

@ -85,6 +85,11 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
sql += " ORDER BY " + makeOrderByClause(table, filter.getOrderBys(), joinsContext);
}
if(aggregateInput.getLimit() != null)
{
sql += " LIMIT " + aggregateInput.getLimit();
}
// todo sql customization - can edit sql and/or param list
AggregateOutput rs = new AggregateOutput();
@ -114,9 +119,18 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(aggregate.getFieldName());
QFieldMetaData field = fieldAndTableNameOrAlias.field();
if(field.getType().equals(QFieldType.INTEGER) && aggregate.getOperator().equals(AggregateOperator.AVG))
QFieldType fieldType = aggregate.getFieldType();
if(fieldType == null)
{
field = new QFieldMetaData().withType(QFieldType.DECIMAL);
if(field.getType().equals(QFieldType.INTEGER) && (aggregate.getOperator().equals(AggregateOperator.AVG)))
{
fieldType = QFieldType.DECIMAL;
}
}
if(fieldType != null)
{
field = new QFieldMetaData().withType(fieldType);
}
Serializable value = getFieldValueFromResultSet(field, resultSet, selectionIndex++);
@ -155,7 +169,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
for(Aggregate aggregate : aggregateInput.getAggregates())
{
JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(aggregate.getFieldName());
rs.add(aggregate.getOperator() + "(" + escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(fieldAndTableNameOrAlias.field())) + ")");
rs.add(aggregate.getOperator().getSqlPrefix() + escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(fieldAndTableNameOrAlias.field())) + ")");
}
return (rs);
}