From 2a684784053910fd91eb20ebee47cb0579a515d1 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 31 Jan 2024 10:58:42 -0600 Subject: [PATCH] Fix how column-stats backend handles date-times, grouping by hour. update MemoryRecordStore to work for an aggregate with a DateTimeGroupBy, at least enough for test to pass. --- .../dashboard/widgets/DateTimeGroupBy.java | 17 ++++ .../memory/MemoryRecordStore.java | 82 +++++++++++++++++-- .../columnstats/ColumnStatsStep.java | 13 ++- .../columnstats/ColumnStatsStepTest.java | 47 +++++++++++ 4 files changed, 147 insertions(+), 12 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java index f69596af..2cc0ba81 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java @@ -297,4 +297,21 @@ public enum DateTimeGroupBy ZonedDateTime zoned = instant.atZone(zoneId); return (zoned.plus(noOfChronoUnitsToAdd, chronoUnitToAdd).toInstant()); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static DateTimeFormatter sqlDateFormatToSelectedDateTimeFormatter(String sqlDateFormat) + { + for(DateTimeGroupBy value : values()) + { + if(value.sqlDateFormat.equals(sqlDateFormat)) + { + return (value.selectedStringFormatter); + } + } + return null; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java index 4685b7e3..890d7c7f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java @@ -24,6 +24,10 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory; import java.io.Serializable; import java.math.BigDecimal; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -35,6 +39,7 @@ import java.util.Objects; import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.DateTimeGroupBy; import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -66,6 +71,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -577,7 +583,11 @@ public class MemoryRecordStore for(GroupBy groupBy : groupBys) { Serializable groupByValue = record.getValue(groupBy.getFieldName()); - if(groupBy.getType() != null) + if(StringUtils.hasContent(groupBy.getFormatString())) + { + groupByValue = applyFormatString(groupByValue, groupBy); + } + else if(groupBy.getType() != null) { groupByValue = ValueUtils.getValueAsFieldType(groupBy.getType(), groupByValue); } @@ -629,7 +639,9 @@ public class MemoryRecordStore ///////////////////// if(aggregateInput.getFilter() != null && CollectionUtils.nullSafeHasContents(aggregateInput.getFilter().getOrderBys())) { - Comparator comparator = null; + ///////////////////////////////////////////////////////////////////////////////////// + // lambda to compare 2 serializables, as we'll assume (& cast) them to Comparables // + ///////////////////////////////////////////////////////////////////////////////////// Comparator serializableComparator = (Serializable a, Serializable b) -> { if(a == null && b == null) @@ -647,9 +659,15 @@ public class MemoryRecordStore return ((Comparable) a).compareTo(b); }; + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // reverse of the lambda above (we had some errors calling .reversed() on the comparator we were building, so this seemed simpler & worked) // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Comparator reverseSerializableComparator = (Serializable a, Serializable b) -> -serializableComparator.compare(a, b); + //////////////////////////////////////////////// // build a comparator out of all the orderBys // //////////////////////////////////////////////// + Comparator comparator = null; for(QFilterOrderBy orderBy : aggregateInput.getFilter().getOrderBys()) { Function keyExtractor = aggregateResult -> @@ -670,16 +688,11 @@ public class MemoryRecordStore if(comparator == null) { - comparator = Comparator.comparing(keyExtractor, serializableComparator); + comparator = Comparator.comparing(keyExtractor, orderBy.getIsAscending() ? serializableComparator : reverseSerializableComparator); } else { - comparator = comparator.thenComparing(keyExtractor, serializableComparator); - } - - if(!orderBy.getIsAscending()) - { - comparator = comparator.reversed(); + comparator = comparator.thenComparing(keyExtractor, orderBy.getIsAscending() ? serializableComparator : reverseSerializableComparator); } } @@ -696,6 +709,57 @@ public class MemoryRecordStore + /******************************************************************************* + ** + *******************************************************************************/ + private Serializable applyFormatString(Serializable value, GroupBy groupBy) throws QException + { + if(value == null) + { + return (null); + } + + String formatString = groupBy.getFormatString(); + + try + { + if(formatString.startsWith("DATE_FORMAT")) + { + ///////////////////////////////////////////////////////////////////////////// + // one known-use case we have here looks like this: // + // DATE_FORMAT(CONVERT_TZ(%s, 'UTC', 'UTC'), '%%Y-%%m-%%dT%%H') // + // ... for now, let's just try to support the formatting bit at the end... // + // todo - support the CONVERT_TZ bit too! // + ///////////////////////////////////////////////////////////////////////////// + String sqlDateTimeFormat = formatString.replaceFirst(".*'%%", "%%").replaceFirst("'.*", ""); + DateTimeFormatter dateTimeFormatter = DateTimeGroupBy.sqlDateFormatToSelectedDateTimeFormatter(sqlDateTimeFormat); + if(dateTimeFormatter == null) + { + throw (new QException("Unsupported sql dateTime format string [" + sqlDateTimeFormat + "] for MemoryRecordStore")); + } + + String valueAsString = ValueUtils.getValueAsString(value); + Instant valueAsInstant = ValueUtils.getValueAsInstant(valueAsString); + ZonedDateTime zonedDateTime = valueAsInstant.atZone(ZoneId.systemDefault()); + return (dateTimeFormatter.format(zonedDateTime)); + } + else + { + throw (new QException("Unsupported group-by format string [" + formatString + "] for MemoryRecordStore")); + } + } + catch(QException qe) + { + throw (qe); + } + catch(Exception e) + { + throw (new QException("Error applying format string [" + formatString + "] to group by value [" + value + "]", e)); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java index bfeb47f4..39dbd7f1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java @@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats; import java.io.Serializable; import java.math.BigDecimal; import java.math.RoundingMode; +import java.time.Instant; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -173,11 +175,10 @@ public class ColumnStatsStep implements BackendStep Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL); GroupBy groupBy = new GroupBy(field.getType(), fieldName); - // todo - something here about "by-date, not time" + // todo - something here about an input param to specify how you want dates & date-times grouped if(field.getType().equals(QFieldType.DATE_TIME)) { - // groupBy = new GroupBy(field.getType(), fieldName, "DATE(%s)"); - String sqlExpression = DateTimeGroupBy.HOUR.getSqlExpression(); + String sqlExpression = DateTimeGroupBy.HOUR.getSqlExpression(ZoneId.systemDefault()); groupBy = new GroupBy(QFieldType.STRING, fieldName, sqlExpression); } @@ -230,6 +231,12 @@ public class ColumnStatsStep implements BackendStep for(AggregateResult result : aggregateOutput.getResults()) { Serializable value = result.getGroupByValue(groupBy); + + if(field.getType().equals(QFieldType.DATE_TIME) && value != null) + { + value = Instant.parse(value + ":00:00Z"); + } + Integer count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate)); valueCounts.add(new QRecord().withValue(fieldName, value).withValue("count", count)); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java index d2b6fa0b..1e59efd9 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats; import java.io.Serializable; import java.math.BigDecimal; +import java.time.Instant; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; @@ -91,4 +92,50 @@ class ColumnStatsStepTest extends BaseTest .hasFieldOrPropertyWithValue("percent", new BigDecimal("16.67")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDateTimesRollupByHour() throws QException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("timestamp", Instant.parse("2024-01-31T09:59:01Z")), + new QRecord().withValue("timestamp", Instant.parse("2024-01-31T09:59:59Z")), + new QRecord().withValue("timestamp", Instant.parse("2024-01-31T10:00:00Z")), + new QRecord().withValue("timestamp", Instant.parse("2024-01-31T10:01:01Z")), + new QRecord().withValue("timestamp", Instant.parse("2024-01-31T10:59:59Z")), + new QRecord().withValue("timestamp", null) + )); + new InsertAction().execute(insertInput); + + RunBackendStepInput input = new RunBackendStepInput(); + input.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); + input.addValue("fieldName", "timestamp"); + input.addValue("orderBy", "count.desc"); + + RunBackendStepOutput output = new RunBackendStepOutput(); + new ColumnStatsStep().run(input, output); + + Map values = output.getValues(); + + @SuppressWarnings("unchecked") + List valueCounts = (List) values.get("valueCounts"); + + assertThat(valueCounts.get(0).getValues()) + .hasFieldOrPropertyWithValue("timestamp", Instant.parse("2024-01-31T10:00:00Z")) + .hasFieldOrPropertyWithValue("count", 3); + + assertThat(valueCounts.get(1).getValues()) + .hasFieldOrPropertyWithValue("timestamp", Instant.parse("2024-01-31T09:00:00Z")) + .hasFieldOrPropertyWithValue("count", 2); + + assertThat(valueCounts.get(2).getValues()) + .hasFieldOrPropertyWithValue("timestamp", null) + .hasFieldOrPropertyWithValue("count", 1); + } + } \ No newline at end of file