Fix how column-stats backend handles date-times, grouping by hour. update MemoryRecordStore to work for an aggregate with a DateTimeGroupBy, at least enough for test to pass.

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java

@@ -297,4 +297,21 @@ public enum DateTimeGroupBy
       ZonedDateTime zoned = instant.atZone(zoneId);
       return (zoned.plus(noOfChronoUnitsToAdd, chronoUnitToAdd).toInstant());
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static DateTimeFormatter sqlDateFormatToSelectedDateTimeFormatter(String sqlDateFormat)
+   {
+      for(DateTimeGroupBy value : values())
+      {
+         if(value.sqlDateFormat.equals(sqlDateFormat))
+         {
+            return (value.selectedStringFormatter);
+         }
+      }
+      return null;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java

@@ -24,6 +24,10 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
 
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -35,6 +39,7 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.DateTimeGroupBy;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -66,6 +71,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
@@ -577,7 +583,11 @@ public class MemoryRecordStore
          for(GroupBy groupBy : groupBys)
          {
             Serializable groupByValue = record.getValue(groupBy.getFieldName());
-            if(groupBy.getType() != null)
+            if(StringUtils.hasContent(groupBy.getFormatString()))
+            {
+               groupByValue = applyFormatString(groupByValue, groupBy);
+            }
+            else if(groupBy.getType() != null)
             {
                groupByValue = ValueUtils.getValueAsFieldType(groupBy.getType(), groupByValue);
             }
@@ -629,7 +639,9 @@ public class MemoryRecordStore
       /////////////////////
       if(aggregateInput.getFilter() != null && CollectionUtils.nullSafeHasContents(aggregateInput.getFilter().getOrderBys()))
       {
-         Comparator<AggregateResult> comparator = null;
+         /////////////////////////////////////////////////////////////////////////////////////
+         // lambda to compare 2 serializables, as we'll assume (& cast) them to Comparables //
+         /////////////////////////////////////////////////////////////////////////////////////
          Comparator<Serializable> serializableComparator = (Serializable a, Serializable b) ->
          {
             if(a == null && b == null)
@@ -647,9 +659,15 @@ public class MemoryRecordStore
             return ((Comparable) a).compareTo(b);
          };
 
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // reverse of the lambda above (we had some errors calling .reversed() on the comparator we were building, so this seemed simpler & worked) //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         Comparator<Serializable> reverseSerializableComparator = (Serializable a, Serializable b) -> -serializableComparator.compare(a, b);
+
          ////////////////////////////////////////////////
          // build a comparator out of all the orderBys //
          ////////////////////////////////////////////////
+         Comparator<AggregateResult> comparator = null;
          for(QFilterOrderBy orderBy : aggregateInput.getFilter().getOrderBys())
          {
             Function<AggregateResult, Serializable> keyExtractor = aggregateResult ->
@@ -670,16 +688,11 @@ public class MemoryRecordStore
 
             if(comparator == null)
             {
-               comparator = Comparator.comparing(keyExtractor, serializableComparator);
+               comparator = Comparator.comparing(keyExtractor, orderBy.getIsAscending() ? serializableComparator : reverseSerializableComparator);
             }
             else
             {
-               comparator = comparator.thenComparing(keyExtractor, serializableComparator);
+               comparator = comparator.thenComparing(keyExtractor, orderBy.getIsAscending() ? serializableComparator : reverseSerializableComparator);
-            }
-
-            if(!orderBy.getIsAscending())
-            {
-               comparator = comparator.reversed();
             }
          }
 
@@ -696,6 +709,57 @@ public class MemoryRecordStore
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Serializable applyFormatString(Serializable value, GroupBy groupBy) throws QException
+   {
+      if(value == null)
+      {
+         return (null);
+      }
+
+      String formatString = groupBy.getFormatString();
+
+      try
+      {
+         if(formatString.startsWith("DATE_FORMAT"))
+         {
+            /////////////////////////////////////////////////////////////////////////////
+            // one known-use case we have here looks like this:                        //
+            // DATE_FORMAT(CONVERT_TZ(%s, 'UTC', 'UTC'), '%%Y-%%m-%%dT%%H')            //
+            // ... for now, let's just try to support the formatting bit at the end... //
+            // todo - support the CONVERT_TZ bit too!                                  //
+            /////////////////////////////////////////////////////////////////////////////
+            String            sqlDateTimeFormat = formatString.replaceFirst(".*'%%", "%%").replaceFirst("'.*", "");
+            DateTimeFormatter dateTimeFormatter = DateTimeGroupBy.sqlDateFormatToSelectedDateTimeFormatter(sqlDateTimeFormat);
+            if(dateTimeFormatter == null)
+            {
+               throw (new QException("Unsupported sql dateTime format string [" + sqlDateTimeFormat + "] for MemoryRecordStore"));
+            }
+
+            String        valueAsString  = ValueUtils.getValueAsString(value);
+            Instant       valueAsInstant = ValueUtils.getValueAsInstant(valueAsString);
+            ZonedDateTime zonedDateTime  = valueAsInstant.atZone(ZoneId.systemDefault());
+            return (dateTimeFormatter.format(zonedDateTime));
+         }
+         else
+         {
+            throw (new QException("Unsupported group-by format string [" + formatString + "] for MemoryRecordStore"));
+         }
+      }
+      catch(QException qe)
+      {
+         throw (qe);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error applying format string [" + formatString + "] to group by value [" + value + "]", e));
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java

@@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
 import java.io.Serializable;
 import java.math.BigDecimal;
 import java.math.RoundingMode;
+import java.time.Instant;
+import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -173,11 +175,10 @@ public class ColumnStatsStep implements BackendStep
          Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
          GroupBy   groupBy   = new GroupBy(field.getType(), fieldName);
 
-         // todo - something here about "by-date, not time"
+         // todo - something here about an input param to specify how you want dates & date-times grouped
          if(field.getType().equals(QFieldType.DATE_TIME))
          {
-            // groupBy = new GroupBy(field.getType(), fieldName, "DATE(%s)");
+            String sqlExpression = DateTimeGroupBy.HOUR.getSqlExpression(ZoneId.systemDefault());
-            String sqlExpression = DateTimeGroupBy.HOUR.getSqlExpression();
             groupBy = new GroupBy(QFieldType.STRING, fieldName, sqlExpression);
          }
 
@@ -230,6 +231,12 @@ public class ColumnStatsStep implements BackendStep
          for(AggregateResult result : aggregateOutput.getResults())
          {
             Serializable value = result.getGroupByValue(groupBy);
+
+            if(field.getType().equals(QFieldType.DATE_TIME) && value != null)
+            {
+               value = Instant.parse(value + ":00:00Z");
+            }
+
             Integer      count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate));
             valueCounts.add(new QRecord().withValue(fieldName, value).withValue("count", count));
          }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
 
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.time.Instant;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.BaseTest;
@@ -91,4 +92,50 @@ class ColumnStatsStepTest extends BaseTest
          .hasFieldOrPropertyWithValue("percent", new BigDecimal("16.67"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testDateTimesRollupByHour() throws QException
+   {
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("timestamp", Instant.parse("2024-01-31T09:59:01Z")),
+         new QRecord().withValue("timestamp", Instant.parse("2024-01-31T09:59:59Z")),
+         new QRecord().withValue("timestamp", Instant.parse("2024-01-31T10:00:00Z")),
+         new QRecord().withValue("timestamp", Instant.parse("2024-01-31T10:01:01Z")),
+         new QRecord().withValue("timestamp", Instant.parse("2024-01-31T10:59:59Z")),
+         new QRecord().withValue("timestamp", null)
+      ));
+      new InsertAction().execute(insertInput);
+
+      RunBackendStepInput input = new RunBackendStepInput();
+      input.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      input.addValue("fieldName", "timestamp");
+      input.addValue("orderBy", "count.desc");
+
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      new ColumnStatsStep().run(input, output);
+
+      Map<String, Serializable> values = output.getValues();
+
+      @SuppressWarnings("unchecked")
+      List<QRecord> valueCounts = (List<QRecord>) values.get("valueCounts");
+
+      assertThat(valueCounts.get(0).getValues())
+         .hasFieldOrPropertyWithValue("timestamp", Instant.parse("2024-01-31T10:00:00Z"))
+         .hasFieldOrPropertyWithValue("count", 3);
+
+      assertThat(valueCounts.get(1).getValues())
+         .hasFieldOrPropertyWithValue("timestamp", Instant.parse("2024-01-31T09:00:00Z"))
+         .hasFieldOrPropertyWithValue("count", 2);
+
+      assertThat(valueCounts.get(2).getValues())
+         .hasFieldOrPropertyWithValue("timestamp", null)
+         .hasFieldOrPropertyWithValue("count", 1);
+   }
+
 }