QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

WIP version of table/column stats process & supporting aggregate changes

Browse Source
This commit is contained in:
darin.kelkhoff
2023-02-27 10:30:54 -06:00
parent 03e9f27866
commit da17145f66
7 changed files with 217 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
View File

@ -51,7 +51,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
@ -293,14 +292,6 @@ public class QInstanceEnricher
*******************************************************************************/
private void enrichStep(QStepMetaData step)
{
if(!StringUtils.hasContent(step.getName()) && step instanceof QBackendStepMetaData backendStep)
{
if(backendStep.getCode() != null && backendStep.getCode().getName() != null)
{
step.setName(backendStep.getCode().getName());
}
}
if(!StringUtils.hasContent(step.getLabel()))
{
step.setLabel(nameToLabel(step.getName()));

35
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateInput.java
View File

@ -38,6 +38,7 @@ public class AggregateInput extends AbstractTableActionInput
private QQueryFilter filter;
private List<Aggregate> aggregates;
private List<GroupBy> groupBys = new ArrayList<>();
private Integer limit;
private List<QueryJoin> queryJoins = null;
@ -234,4 +235,38 @@ public class AggregateInput extends AbstractTableActionInput
return (this);
}
/*******************************************************************************
** Getter for limit
**
*******************************************************************************/
public Integer getLimit()
{
return limit;
}
/*******************************************************************************
** Setter for limit
**
*******************************************************************************/
public void setLimit(Integer limit)
{
this.limit = limit;
}
/*******************************************************************************
** Fluent setter for limit
**
*******************************************************************************/
public AggregateInput withLimit(Integer limit)
{
this.limit = limit;
return (this);
}
}

34
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateOperator.java
View File

@ -27,9 +27,33 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.aggregate;
*******************************************************************************/
public enum AggregateOperator
{
COUNT,
SUM,
MIN,
MAX,
AVG
COUNT("COUNT("),
COUNT_DISTINCT("COUNT(DISTINCT "),
SUM("SUM("),
MIN("MIN("),
MAX("MAX("),
AVG("AVG(");
private final String sqlPrefix;
/*******************************************************************************
**
*******************************************************************************/
AggregateOperator(String sqlPrefix)
{
this.sqlPrefix = sqlPrefix;
}
/*******************************************************************************
** Getter for sqlPrefix
**
*******************************************************************************/
public String getSqlPrefix()
{
return sqlPrefix;
}
}

11
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java
View File

@ -38,6 +38,17 @@ public class GroupBy implements Serializable
/*******************************************************************************
**
*******************************************************************************/
public GroupBy(QFieldType type, String fieldName)
{
this.type = type;
this.fieldName = fieldName;
}
/*******************************************************************************
**
*******************************************************************************/

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java
View File

@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
@ -234,6 +235,12 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
{
this.steps = new HashMap<>();
}
if(!StringUtils.hasContent(step.getName()))
{
throw (new IllegalArgumentException("Attempt to add a process step without a name"));
}
this.steps.put(step.getName(), step);
return (this);
@ -251,6 +258,12 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
{
this.steps = new HashMap<>();
}
if(!StringUtils.hasContent(step.getName()))
{
throw (new IllegalArgumentException("Attempt to add a process step without a name"));
}
this.steps.put(step.getName(), step);
return (this);

123
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java Normal file
View File

@ -0,0 +1,123 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.tablestats;
import java.io.Serializable;
import java.util.ArrayList;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
**
*******************************************************************************/
public class TableStatsStep implements BackendStep
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
try
{
String tableName = runBackendStepInput.getValueString("tableName");
String fieldName = runBackendStepInput.getValueString("fieldName");
String filterJSON = runBackendStepInput.getValueString("filterJSON");
QQueryFilter filter = null;
if(StringUtils.hasContent(filterJSON))
{
filter = JsonUtils.toObject(filterJSON, QQueryFilter.class);
}
else
{
filter = new QQueryFilter();
}
QTableMetaData table = QContext.getQInstance().getTable(tableName);
QFieldMetaData field = table.getField(fieldName);
Aggregate aggregate = new Aggregate(fieldName, AggregateOperator.COUNT);
GroupBy groupBy = new GroupBy(field.getType(), fieldName);
Integer limit = 1000; // too big?
AggregateInput aggregateInput = new AggregateInput();
aggregateInput.withAggregate(aggregate);
aggregateInput.withGroupBy(groupBy);
aggregateInput.setTableName(tableName);
aggregateInput.setFilter(filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)));
aggregateInput.setLimit(limit);
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
ArrayList<QRecord> valueCounts = new ArrayList<>();
for(AggregateResult result : aggregateOutput.getResults())
{
Serializable value = result.getGroupByValue(groupBy);
Integer count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate));
valueCounts.add(new QRecord().withValue("value", value).withValue("count", count));
}
runBackendStepOutput.addValue("valueCounts", valueCounts);
if(valueCounts.size() < limit)
{
runBackendStepOutput.addValue("countDistinct", valueCounts.size());
}
else
{
Aggregate countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT);
AggregateInput countDistinctAggregateInput = new AggregateInput();
countDistinctAggregateInput.withAggregate(countDistinctAggregate);
countDistinctAggregateInput.setTableName(tableName);
countDistinctAggregateInput.setFilter(filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)));
AggregateOutput countDistinctAggregateOutput = new AggregateAction().execute(countDistinctAggregateInput);
AggregateResult countDistinctAggregateResult = countDistinctAggregateOutput.getResults().get(0);
runBackendStepOutput.addValue("countDistinct", countDistinctAggregateResult.getAggregateValue(countDistinctAggregate));
}
}
catch(Exception e)
{
throw new QException("Error calculating stats", e);
}
}
}

7
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java
View File

@ -85,6 +85,11 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
sql += " ORDER BY " + makeOrderByClause(table, filter.getOrderBys(), joinsContext);
}
if(aggregateInput.getLimit() != null)
{
sql += " LIMIT " + aggregateInput.getLimit();
}
// todo sql customization - can edit sql and/or param list
AggregateOutput rs = new AggregateOutput();
@ -155,7 +160,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
for(Aggregate aggregate : aggregateInput.getAggregates())
{
JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(aggregate.getFieldName());
rs.add(aggregate.getOperator() + "(" + escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(fieldAndTableNameOrAlias.field())) + ")");
rs.add(aggregate.getOperator().getSqlPrefix() + escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(fieldAndTableNameOrAlias.field())) + ")");
}
return (rs);
}