CE-1328 Add job status messages

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java

@@ -301,9 +301,6 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
             }
          }
 
-         widgetData.setAllowRecordEdit(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("allowRecordEdit"))));
-         widgetData.setAllowRecordDelete(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("allowRecordDelete"))));
-
          return (new RenderWidgetOutput(widgetData));
       }
       catch(Exception e)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ProcessAlertWidget.java

@@ -40,9 +40,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
  ** - alertType - name of entry in AlertType enum (ERROR, WARNING, SUCCESS)
  ** - alertHtml - html to display inside the alert (other than its icon)
  *******************************************************************************/
-public class AlertWidgetRenderer extends AbstractWidgetRenderer implements MetaDataProducerInterface<QWidgetMetaData>
+public class ProcessAlertWidget extends AbstractWidgetRenderer implements MetaDataProducerInterface<QWidgetMetaData>
 {
-   public static final String NAME = "AlertWidgetRenderer";
+   public static final String NAME = "ProcessAlertWidget";
 
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/AlertData.java

@@ -22,9 +22,6 @@
 package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
 
 
-import java.util.List;
-
-
 /*******************************************************************************
  ** Model containing datastructure expected by frontend alert widget
  **
@@ -43,10 +40,8 @@ public class AlertData extends QWidgetData
 
 
 
-   private String       html;
+   private String    html;
-   private AlertType    alertType;
+   private AlertType alertType;
-   private Boolean      hideWidget = false;
-   private List<String> bulletList;
 
 
 
@@ -144,66 +139,4 @@ public class AlertData extends QWidgetData
       return (this);
    }
 
-
-
-   /*******************************************************************************
-    ** Getter for hideWidget
-    *******************************************************************************/
-   public boolean getHideWidget()
-   {
-      return (this.hideWidget);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for hideWidget
-    *******************************************************************************/
-   public void setHideWidget(boolean hideWidget)
-   {
-      this.hideWidget = hideWidget;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for hideWidget
-    *******************************************************************************/
-   public AlertData withHideWidget(boolean hideWidget)
-   {
-      this.hideWidget = hideWidget;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for bulletList
-    *******************************************************************************/
-   public List<String> getBulletList()
-   {
-      return (this.bulletList);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for bulletList
-    *******************************************************************************/
-   public void setBulletList(List<String> bulletList)
-   {
-      this.bulletList = bulletList;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for bulletList
-    *******************************************************************************/
-   public AlertData withBulletList(List<String> bulletList)
-   {
-      this.bulletList = bulletList;
-      return (this);
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChildRecordListData.java

@@ -39,13 +39,9 @@ public class ChildRecordListData extends QWidgetData
    private QueryOutput    queryOutput;
    private QTableMetaData childTableMetaData;
 
-   private String  tableName;
    private String  tablePath;
    private String  viewAllLink;
    private Integer totalRows;
-   private Boolean disableRowClick   = false;
-   private Boolean allowRecordEdit   = false;
-   private Boolean allowRecordDelete = false;
 
    private boolean                   canAddChildRecord = false;
    private Map<String, Serializable> defaultValuesForNewChildRecords;
@@ -356,141 +352,4 @@ public class ChildRecordListData extends QWidgetData
       return (this);
    }
 
-
-
-   /*******************************************************************************
-    ** Getter for tableName
-    *******************************************************************************/
-   public String getTableName()
-   {
-      return (this.tableName);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for tableName
-    *******************************************************************************/
-   public void setTableName(String tableName)
-   {
-      this.tableName = tableName;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for tableName
-    *******************************************************************************/
-   public ChildRecordListData withTableName(String tableName)
-   {
-      this.tableName = tableName;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for tablePath
-    *******************************************************************************/
-   public ChildRecordListData withTablePath(String tablePath)
-   {
-      this.tablePath = tablePath;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for disableRowClick
-    *******************************************************************************/
-   public Boolean getDisableRowClick()
-   {
-      return (this.disableRowClick);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for disableRowClick
-    *******************************************************************************/
-   public void setDisableRowClick(Boolean disableRowClick)
-   {
-      this.disableRowClick = disableRowClick;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for disableRowClick
-    *******************************************************************************/
-   public ChildRecordListData withDisableRowClick(Boolean disableRowClick)
-   {
-      this.disableRowClick = disableRowClick;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for allowRecordEdit
-    *******************************************************************************/
-   public Boolean getAllowRecordEdit()
-   {
-      return (this.allowRecordEdit);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for allowRecordEdit
-    *******************************************************************************/
-   public void setAllowRecordEdit(Boolean allowRecordEdit)
-   {
-      this.allowRecordEdit = allowRecordEdit;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for allowRecordEdit
-    *******************************************************************************/
-   public ChildRecordListData withAllowRecordEdit(Boolean allowRecordEdit)
-   {
-      this.allowRecordEdit = allowRecordEdit;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for allowRecordDelete
-    *******************************************************************************/
-   public Boolean getAllowRecordDelete()
-   {
-      return (this.allowRecordDelete);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for allowRecordDelete
-    *******************************************************************************/
-   public void setAllowRecordDelete(Boolean allowRecordDelete)
-   {
-      this.allowRecordDelete = allowRecordDelete;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for allowRecordDelete
-    *******************************************************************************/
-   public ChildRecordListData withAllowRecordDelete(Boolean allowRecordDelete)
-   {
-      this.allowRecordDelete = allowRecordDelete;
-      return (this);
-   }
 }
-
-
-

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidgetData.java

@@ -31,7 +31,7 @@ import java.util.Map;
  ** Base class for the data returned by rendering a Widget.
  **
  *******************************************************************************/
-public abstract class QWidgetData implements Serializable
+public abstract class QWidgetData
 {
    private String       label;
    private String       sublabel;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GenericGarbageCollectorExecuteStep.java

@@ -0,0 +1,287 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class GenericGarbageCollectorExecuteStep implements BackendStep
+{
+   private static final QLogger LOG = QLogger.getLogger(GenericGarbageCollectorExecuteStep.class);
+
+   private ProcessSummaryLine partitionsLine = new ProcessSummaryLine(Status.INFO)
+      .withSingularPastMessage("partition was processed")
+      .withPluralPastMessage("partitions were processed");
+
+   private ProcessSummaryLine deletedLine = new ProcessSummaryLine(Status.OK)
+      .withSingularPastMessage("record was deleted")
+      .withPluralPastMessage("records were deleted");
+
+   private ProcessSummaryLine warningLine = new ProcessSummaryLine(Status.WARNING, "had an warning");
+   private ProcessSummaryLine errorLine   = new ProcessSummaryLine(Status.ERROR, "had an error");
+
+   private AsyncJobCallback asyncJobCallback;
+
+   private Integer total        = null;
+   private Integer deletedSoFar = 0;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      String         tableName = runBackendStepInput.getValueString("table");
+      QTableMetaData table     = QContext.getQInstance().getTable(tableName);
+      if(table == null)
+      {
+         throw new QUserFacingException("Unrecognized table: " + tableName);
+      }
+
+      String         fieldName = runBackendStepInput.getValueString("field");
+      QFieldMetaData field     = table.getFields().get(fieldName);
+      if(field == null)
+      {
+         throw new QUserFacingException("Unrecognized field: " + fieldName);
+      }
+
+      if(!QFieldType.DATE_TIME.equals(field.getType()) && !QFieldType.DATE.equals(field.getType()))
+      {
+         throw new QUserFacingException("Field " + field + " is not a date-time or date type field.");
+      }
+
+      Integer daysBack = runBackendStepInput.getValueInteger("daysBack");
+      if(daysBack == null || daysBack < 0)
+      {
+         throw new QUserFacingException("Illegal value for daysBack: " + daysBack + "; Must be positive.");
+      }
+
+      Integer maxPageSize = runBackendStepInput.getValueInteger("maxPageSize");
+      if(maxPageSize == null || maxPageSize < 0)
+      {
+         throw new QUserFacingException("Illegal value for maxPageSize: " + maxPageSize + "; Must be positive.");
+      }
+
+      asyncJobCallback = runBackendStepInput.getAsyncJobCallback();
+      long start = System.currentTimeMillis();
+      execute(table, field, daysBack, maxPageSize);
+      long end = System.currentTimeMillis();
+
+      deletedLine.prepareForFrontend(true);
+      partitionsLine.prepareForFrontend(true);
+
+      ArrayList<ProcessSummaryLineInterface> processSummary = new ArrayList<>();
+      processSummary.add(partitionsLine);
+      processSummary.add(deletedLine);
+      warningLine.addSelfToListIfAnyCount(processSummary);
+      errorLine.addSelfToListIfAnyCount(processSummary);
+      processSummary.add(new ProcessSummaryLine(Status.INFO, "Total time:  " + String.format("%,d", ((end - start) / 1000)) + " seconds"));
+      runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummary);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void execute(QTableMetaData table, QFieldMetaData field, Integer daysBack, Integer maxPageSize) throws QException
+   {
+      asyncJobCallback.updateStatus("Counting records");
+      Instant maxDate = Instant.now().minusSeconds(daysBack * 60 * 60 * 24);
+      Instant minDate = findMinDateInTable(table, field);
+
+      processDateRange(table, field, maxPageSize, minDate, maxDate);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void processDateRange(QTableMetaData table, QFieldMetaData field, Integer maxPageSize, Instant minDate, Instant maxDate) throws QException
+   {
+      partitionsLine.incrementCount();
+
+      LOG.info("Counting", logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+      Integer count = count(table, field, minDate, maxDate);
+      LOG.info("Count", logPair("count", count), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+
+      if(this.total == null)
+      {
+         this.total = count;
+      }
+
+      if(count == 0)
+      {
+         LOG.info("0 rows in this partition - nothing to delete", logPair("count", count), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+      }
+      else if(count <= maxPageSize)
+      {
+         asyncJobCallback.updateStatus("Deleting records", deletedSoFar, total);
+         LOG.info("Deleting", logPair("count", count), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+         delete(table, field, minDate, maxDate);
+         this.deletedSoFar += count;
+      }
+      else
+      {
+         if(deletedSoFar == 0)
+         {
+            asyncJobCallback.updateStatus("Partitioning table", deletedSoFar, total);
+         }
+
+         LOG.info("Too many rows", logPair("count", count), logPair("maxPageSize", maxPageSize), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+         partition(table, field, minDate, maxDate, count, maxPageSize);
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void partition(QTableMetaData table, QFieldMetaData field, Instant minDate, Instant maxDate, Integer count, Integer maxPageSize) throws QException
+   {
+      int  noOfPartitions    = (int) Math.ceil((float) count / (float) maxPageSize);
+      long milliDiff         = maxDate.toEpochMilli() - minDate.toEpochMilli();
+      long milliPerPartition = milliDiff / noOfPartitions;
+
+      if(milliPerPartition < 1000)
+      {
+         throw (new QUserFacingException("To find a maxPageSize under " + String.format("%,d", maxPageSize) + ", the partition size would become smaller than 1 second (between " + minDate + " and " + maxDate + " there are " + String.format("%,d", count) + " rows) - you must use a larger maxPageSize to continue."));
+      }
+
+      LOG.info("Partitioning", logPair("count", count), logPair("noOfPartitions", noOfPartitions), logPair("milliDiff", milliDiff), logPair("milliPerPartition", milliPerPartition), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+      for(int i = 0; i < noOfPartitions; i++)
+      {
+         maxDate = minDate.plusMillis(milliPerPartition);
+         processDateRange(table, field, maxPageSize, minDate, maxDate);
+         minDate = maxDate;
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void delete(QTableMetaData table, QFieldMetaData field, Instant minDate, Instant maxDate) throws QException
+   {
+      DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(table.getName())
+         .withQueryFilter(new QQueryFilter(new QFilterCriteria(field.getName(), QCriteriaOperator.BETWEEN, minDate, maxDate))));
+
+      deletedLine.incrementCount(deleteOutput.getDeletedRecordCount());
+
+      if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors()))
+      {
+         warningLine.incrementCount(deleteOutput.getRecordsWithWarnings().size());
+      }
+
+      if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors()))
+      {
+         errorLine.incrementCount(deleteOutput.getRecordsWithErrors().size());
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private Integer count(QTableMetaData table, QFieldMetaData field, Instant minDate, Instant maxDate) throws QException
+   {
+      Aggregate      countDateField = new Aggregate(field.getName(), AggregateOperator.COUNT).withFieldType(QFieldType.INTEGER);
+      AggregateInput aggregateInput = new AggregateInput();
+      aggregateInput.setTableName(table.getName());
+      aggregateInput.withFilter(new QQueryFilter(new QFilterCriteria(field.getName(), QCriteriaOperator.BETWEEN, minDate, maxDate)));
+      aggregateInput.withAggregate(countDateField);
+
+      AggregateOutput       aggregateOutput = new AggregateAction().execute(aggregateInput);
+      List<AggregateResult> results         = aggregateOutput.getResults();
+      if(CollectionUtils.nullSafeIsEmpty(results))
+      {
+         throw (new QUserFacingException("Could not count rows table (null or empty aggregate result)."));
+      }
+
+      return (ValueUtils.getValueAsInteger(results.get(0).getAggregateValue(countDateField)));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private Instant findMinDateInTable(QTableMetaData table, QFieldMetaData field) throws QException
+   {
+      Aggregate      minDate        = new Aggregate(field.getName(), AggregateOperator.MIN);
+      AggregateInput aggregateInput = new AggregateInput();
+      aggregateInput.setTableName(table.getName());
+      aggregateInput.withAggregate(minDate);
+
+      AggregateOutput       aggregateOutput = new AggregateAction().execute(aggregateInput);
+      List<AggregateResult> results         = aggregateOutput.getResults();
+      if(CollectionUtils.nullSafeIsEmpty(results))
+      {
+         throw (new QUserFacingException("Could not find min date value in table (null or empty aggregate result)."));
+      }
+
+      return (ValueUtils.getValueAsInstant(results.get(0).getAggregateValue(minDate)));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GenericGarbageCollectorProcessMetaDataProducer.java

@@ -0,0 +1,88 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+
+
+/*******************************************************************************
+ ** Generic process that can perform garbage collection on any table (at least,
+ ** any table with a date or date-time field).
+ **
+ ** When running, this process prompts for:
+ ** - table name
+ ** - field name (e.g., the date/date-time field on that table)
+ ** - daysBack - any records older than that many days ago will be deleted.
+ ** - maxPageSize - to avoid running "1 huge query", if there are more than
+ ** this number of records between the min-date in the table and the max-date
+ ** (based on daysBack), then the time range is partitioned recursively until
+ ** pages smaller than this parameter are found.  The partitioning attempts to
+ ** be smart (e.g., not just ÷ 2), by doing count / maxPageSize.
+ *******************************************************************************/
+public class GenericGarbageCollectorProcessMetaDataProducer extends MetaDataProducer<QProcessMetaData>
+{
+   public static final String NAME = "GenericGarbageCollector";
+
+
+
+   /*******************************************************************************
+    ** See class header for param descriptions.
+    *******************************************************************************/
+   @Override
+   public QProcessMetaData produce(QInstance qInstance) throws QException
+   {
+      QProcessMetaData processMetaData = new QProcessMetaData()
+         .withName(NAME)
+         .withIcon(new QIcon().withName("auto_delete"))
+         .withStepList(List.of(
+            new QFrontendStepMetaData()
+               .withName("input")
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+               .withFormField(new QFieldMetaData("table", QFieldType.STRING))
+               .withFormField(new QFieldMetaData("field", QFieldType.STRING))
+               .withFormField(new QFieldMetaData("daysBack", QFieldType.INTEGER).withDefaultValue(90))
+               .withFormField(new QFieldMetaData("maxPageSize", QFieldType.INTEGER).withDefaultValue(100000)),
+            new QBackendStepMetaData()
+               .withName("execute")
+               .withCode(new QCodeReference(GenericGarbageCollectorExecuteStep.class)),
+            new QFrontendStepMetaData()
+               .withName("result")
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.PROCESS_SUMMARY_RESULTS))
+         ));
+
+      return (processMetaData);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CountingHash.java

@@ -94,29 +94,6 @@ public class CountingHash<K extends Serializable> extends AbstractMap<K, Integer
 
 
 
-   /*******************************************************************************
-    ** increment the value for the specified key
-    **
-    *******************************************************************************/
-   public Integer put(K key)
-   {
-      return (add(key));
-   }
-
-
-
-   /*******************************************************************************
-    ** Set the value for the specified key by the supplied value
-    **
-    *******************************************************************************/
-   public Integer put(K key, Integer value)
-   {
-      this.map.put(key, value);
-      return (value);
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ProcessAlertWidgetTest.java

@@ -35,9 +35,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
 /*******************************************************************************
- ** Unit test for AlertWidgetRenderer
+ ** Unit test for ProcessAlertWidget 
  *******************************************************************************/
-class AlertWidgetRendererTest extends BaseTest
+class ProcessAlertWidgetTest extends BaseTest
 {
 
    /*******************************************************************************
@@ -46,10 +46,10 @@ class AlertWidgetRendererTest extends BaseTest
    @Test
    void test() throws QException
    {
-      MetaDataProducerHelper.processAllMetaDataProducersInPackage(QContext.getQInstance(), AlertWidgetRenderer.class.getPackageName());
+      MetaDataProducerHelper.processAllMetaDataProducersInPackage(QContext.getQInstance(), ProcessAlertWidget.class.getPackageName());
 
       RenderWidgetInput input = new RenderWidgetInput();
-      input.setWidgetMetaData(QContext.getQInstance().getWidget(AlertWidgetRenderer.NAME));
+      input.setWidgetMetaData(QContext.getQInstance().getWidget(ProcessAlertWidget.NAME));
 
       ///////////////////////////////////////////////////////////////////////////////////////////
       // make sure we run w/o exceptions (and w/ default outputs) if there are no query params //

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GenericGarbageCollectorExecuteStepTest.java

@@ -0,0 +1,243 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
+
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for GenericGarbageCollectorExecuteStep 
+ *******************************************************************************/
+class GenericGarbageCollectorExecuteStepTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   @AfterEach
+   void beforeAndAfterEach()
+   {
+      MemoryRecordStore.getInstance().reset();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testErrors() throws Exception
+   {
+      QContext.getQInstance().addProcess(new GenericGarbageCollectorProcessMetaDataProducer().produce(QContext.getQInstance()));
+
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName(GenericGarbageCollectorProcessMetaDataProducer.NAME);
+      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized table: null");
+
+      input.addValue("table", "notATable");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized table: notATable");
+
+      input.addValue("table", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized field: null");
+
+      input.addValue("field", "notAField");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized field: notAField");
+
+      input.addValue("field", "firstName");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("not a date");
+
+      input.addValue("field", "timestamp");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("daysBack: null");
+
+      input.addValue("daysBack", "-1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("daysBack: -1");
+
+      input.addValue("daysBack", "1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("maxPageSize: null");
+
+      input.addValue("maxPageSize", "-1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("maxPageSize: -1");
+
+      input.addValue("maxPageSize", "1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Could not find min date value in table");
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void runAndThrow(RunProcessInput input) throws Exception
+   {
+      input.setStartAfterStep(null);
+      input.setProcessUUID(null);
+      RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
+      if(runProcessOutput.getException().isPresent())
+      {
+         throw (runProcessOutput.getException().get());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test30days() throws QException
+   {
+      insertAndRunGC(30);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(2, queryOutput.getRecords().size());
+      assertEquals(Set.of(4, 5), queryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   @Disabled("memory aggregator is failing to return an aggregate when no rows found, which is throwing an error...")
+   void test100days() throws QException
+   {
+      insertAndRunGC(100);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(5, queryOutput.getRecords().size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test10days() throws QException
+   {
+      insertAndRunGC(10);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(1, queryOutput.getRecords().size());
+      assertEquals(Set.of(5), queryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test1day() throws QException
+   {
+      insertAndRunGC(1);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(0, queryOutput.getRecords().size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test1dayPartitioned() throws QException
+   {
+      insertAndRunGC(1, 2);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(0, queryOutput.getRecords().size());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void insertAndRunGC(Integer daysBack) throws QException
+   {
+      insertAndRunGC(daysBack, 1000);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void insertAndRunGC(Integer daysBack, Integer maxPageSize) throws QException
+   {
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(getPersonRecords()));
+      QContext.getQInstance().addProcess(new GenericGarbageCollectorProcessMetaDataProducer().produce(QContext.getQInstance()));
+
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName(GenericGarbageCollectorProcessMetaDataProducer.NAME);
+      input.addValue("table", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      input.addValue("field", "timestamp");
+      input.addValue("daysBack", daysBack);
+      input.addValue("maxPageSize", maxPageSize);
+      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+      new RunProcessAction().execute(input);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QRecord> getPersonRecords()
+   {
+      List<QRecord> records = List.of(
+         new QRecord().withValue("id", 1).withValue("timestamp", Instant.now().minus(90, ChronoUnit.DAYS)),
+         new QRecord().withValue("id", 2).withValue("timestamp", Instant.now().minus(31, ChronoUnit.DAYS)),
+         new QRecord().withValue("id", 3).withValue("timestamp", Instant.now().minus(30, ChronoUnit.DAYS).minus(5, ChronoUnit.MINUTES)),
+         new QRecord().withValue("id", 4).withValue("timestamp", Instant.now().minus(29, ChronoUnit.DAYS).minus(23, ChronoUnit.HOURS)),
+         new QRecord().withValue("id", 5).withValue("timestamp", Instant.now().minus(5, ChronoUnit.DAYS)));
+      return records;
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/CountingHashTest.java

@@ -73,19 +73,4 @@ class CountingHashTest extends BaseTest
       assertEquals(1, alwaysMutable.get("B"));
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void testPut()
-   {
-      CountingHash<String> alwaysMutable = new CountingHash<>(Map.of("A", 5));
-      alwaysMutable.put("A", 25);
-      assertEquals(25, alwaysMutable.get("A"));
-      alwaysMutable.put("A");
-      assertEquals(26, alwaysMutable.get("A"));
-   }
-
 }