WIP version of table/column stats process & supporting aggregate changes

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -51,7 +51,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
@@ -293,14 +292,6 @@ public class QInstanceEnricher
     *******************************************************************************/
    private void enrichStep(QStepMetaData step)
    {
-      if(!StringUtils.hasContent(step.getName()) && step instanceof QBackendStepMetaData backendStep)
-      {
-         if(backendStep.getCode() != null && backendStep.getCode().getName() != null)
-         {
-            step.setName(backendStep.getCode().getName());
-         }
-      }
-
       if(!StringUtils.hasContent(step.getLabel()))
       {
          step.setLabel(nameToLabel(step.getName()));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateInput.java

@@ -38,6 +38,7 @@ public class AggregateInput extends AbstractTableActionInput
    private QQueryFilter    filter;
    private List<Aggregate> aggregates;
    private List<GroupBy>   groupBys = new ArrayList<>();
+   private Integer         limit;
 
    private List<QueryJoin> queryJoins = null;
 
@@ -234,4 +235,38 @@ public class AggregateInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for limit
+    **
+    *******************************************************************************/
+   public Integer getLimit()
+   {
+      return limit;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for limit
+    **
+    *******************************************************************************/
+   public void setLimit(Integer limit)
+   {
+      this.limit = limit;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for limit
+    **
+    *******************************************************************************/
+   public AggregateInput withLimit(Integer limit)
+   {
+      this.limit = limit;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateOperator.java

@@ -27,9 +27,33 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.aggregate;
  *******************************************************************************/
 public enum AggregateOperator
 {
-   COUNT,
+   COUNT("COUNT("),
-   SUM,
+   COUNT_DISTINCT("COUNT(DISTINCT "),
-   MIN,
+   SUM("SUM("),
-   MAX,
+   MIN("MIN("),
-   AVG
+   MAX("MAX("),
+   AVG("AVG(");
+
+   private final String sqlPrefix;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   AggregateOperator(String sqlPrefix)
+   {
+      this.sqlPrefix = sqlPrefix;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for sqlPrefix
+    **
+    *******************************************************************************/
+   public String getSqlPrefix()
+   {
+      return sqlPrefix;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java

@@ -38,6 +38,17 @@ public class GroupBy implements Serializable
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public GroupBy(QFieldType type, String fieldName)
+   {
+      this.type = type;
+      this.fieldName = fieldName;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java

@@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
 import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
@@ -234,6 +235,12 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
       {
          this.steps = new HashMap<>();
       }
+
+      if(!StringUtils.hasContent(step.getName()))
+      {
+         throw (new IllegalArgumentException("Attempt to add a process step without a name"));
+      }
+
       this.steps.put(step.getName(), step);
 
       return (this);
@@ -251,6 +258,12 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
       {
          this.steps = new HashMap<>();
       }
+
+      if(!StringUtils.hasContent(step.getName()))
+      {
+         throw (new IllegalArgumentException("Attempt to add a process step without a name"));
+      }
+
       this.steps.put(step.getName(), step);
 
       return (this);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablestats/TableStatsStep.java

@@ -0,0 +1,123 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.tablestats;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class TableStatsStep implements BackendStep
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      try
+      {
+         String tableName  = runBackendStepInput.getValueString("tableName");
+         String fieldName  = runBackendStepInput.getValueString("fieldName");
+         String filterJSON = runBackendStepInput.getValueString("filterJSON");
+
+         QQueryFilter filter = null;
+         if(StringUtils.hasContent(filterJSON))
+         {
+            filter = JsonUtils.toObject(filterJSON, QQueryFilter.class);
+         }
+         else
+         {
+            filter = new QQueryFilter();
+         }
+
+         QTableMetaData table = QContext.getQInstance().getTable(tableName);
+         QFieldMetaData field = table.getField(fieldName);
+
+         Aggregate aggregate = new Aggregate(fieldName, AggregateOperator.COUNT);
+         GroupBy   groupBy   = new GroupBy(field.getType(), fieldName);
+
+         Integer        limit          = 1000; // too big?
+         AggregateInput aggregateInput = new AggregateInput();
+         aggregateInput.withAggregate(aggregate);
+         aggregateInput.withGroupBy(groupBy);
+         aggregateInput.setTableName(tableName);
+         aggregateInput.setFilter(filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)));
+         aggregateInput.setLimit(limit);
+         AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+
+         ArrayList<QRecord> valueCounts = new ArrayList<>();
+         for(AggregateResult result : aggregateOutput.getResults())
+         {
+            Serializable value = result.getGroupByValue(groupBy);
+            Integer      count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate));
+            valueCounts.add(new QRecord().withValue("value", value).withValue("count", count));
+         }
+         runBackendStepOutput.addValue("valueCounts", valueCounts);
+
+         if(valueCounts.size() < limit)
+         {
+            runBackendStepOutput.addValue("countDistinct", valueCounts.size());
+         }
+         else
+         {
+            Aggregate      countDistinctAggregate      = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT);
+            AggregateInput countDistinctAggregateInput = new AggregateInput();
+            countDistinctAggregateInput.withAggregate(countDistinctAggregate);
+            countDistinctAggregateInput.setTableName(tableName);
+            countDistinctAggregateInput.setFilter(filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)));
+            AggregateOutput countDistinctAggregateOutput = new AggregateAction().execute(countDistinctAggregateInput);
+            AggregateResult countDistinctAggregateResult = countDistinctAggregateOutput.getResults().get(0);
+            runBackendStepOutput.addValue("countDistinct", countDistinctAggregateResult.getAggregateValue(countDistinctAggregate));
+         }
+      }
+      catch(Exception e)
+      {
+         throw new QException("Error calculating stats", e);
+      }
+   }
+
+}

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java

@@ -85,6 +85,11 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
             sql += " ORDER BY " + makeOrderByClause(table, filter.getOrderBys(), joinsContext);
          }
 
+         if(aggregateInput.getLimit() != null)
+         {
+            sql += " LIMIT " + aggregateInput.getLimit();
+         }
+
          // todo sql customization - can edit sql and/or param list
 
          AggregateOutput       rs      = new AggregateOutput();
@@ -155,7 +160,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
       for(Aggregate aggregate : aggregateInput.getAggregates())
       {
          JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(aggregate.getFieldName());
-         rs.add(aggregate.getOperator() + "(" + escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(fieldAndTableNameOrAlias.field())) + ")");
+         rs.add(aggregate.getOperator().getSqlPrefix() + escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(fieldAndTableNameOrAlias.field())) + ")");
       }
       return (rs);
    }