QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 21:20:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix how column-stats backend handles date-times, grouping by hour. update MemoryRecordStore to work for an aggregate with a DateTimeGroupBy, at least enough for test to pass.

Browse Source
This commit is contained in:
darin.kelkhoff
2024-01-31 10:58:42 -06:00
parent 08e14ac346
commit 2a68478405
4 changed files with 147 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java
View File

@ -297,4 +297,21 @@ public enum DateTimeGroupBy
ZonedDateTime zoned = instant.atZone(zoneId);
return (zoned.plus(noOfChronoUnitsToAdd, chronoUnitToAdd).toInstant());
}
/*******************************************************************************
**
*******************************************************************************/
public static DateTimeFormatter sqlDateFormatToSelectedDateTimeFormatter(String sqlDateFormat)
{
for(DateTimeGroupBy value : values())
{
if(value.sqlDateFormat.equals(sqlDateFormat))
{
return (value.selectedStringFormatter);
}
}
return null;
}
}

82
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java
View File

@ -24,6 +24,10 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
import java.io.Serializable;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@ -35,6 +39,7 @@ import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.DateTimeGroupBy;
import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -66,6 +71,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ListingHash;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -577,7 +583,11 @@ public class MemoryRecordStore
for(GroupBy groupBy : groupBys)
{
Serializable groupByValue = record.getValue(groupBy.getFieldName());
if(groupBy.getType() != null)
if(StringUtils.hasContent(groupBy.getFormatString()))
{
groupByValue = applyFormatString(groupByValue, groupBy);
}
else if(groupBy.getType() != null)
{
groupByValue = ValueUtils.getValueAsFieldType(groupBy.getType(), groupByValue);
}
@ -629,7 +639,9 @@ public class MemoryRecordStore
/////////////////////
if(aggregateInput.getFilter() != null && CollectionUtils.nullSafeHasContents(aggregateInput.getFilter().getOrderBys()))
{
Comparator<AggregateResult> comparator = null;
/////////////////////////////////////////////////////////////////////////////////////
// lambda to compare 2 serializables, as we'll assume (& cast) them to Comparables //
/////////////////////////////////////////////////////////////////////////////////////
Comparator<Serializable> serializableComparator = (Serializable a, Serializable b) ->
{
if(a == null && b == null)
@ -647,9 +659,15 @@ public class MemoryRecordStore
return ((Comparable) a).compareTo(b);
};
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// reverse of the lambda above (we had some errors calling .reversed() on the comparator we were building, so this seemed simpler & worked) //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Comparator<Serializable> reverseSerializableComparator = (Serializable a, Serializable b) -> -serializableComparator.compare(a, b);
////////////////////////////////////////////////
// build a comparator out of all the orderBys //
////////////////////////////////////////////////
Comparator<AggregateResult> comparator = null;
for(QFilterOrderBy orderBy : aggregateInput.getFilter().getOrderBys())
{
Function<AggregateResult, Serializable> keyExtractor = aggregateResult ->
@ -670,16 +688,11 @@ public class MemoryRecordStore
if(comparator == null)
{
comparator = Comparator.comparing(keyExtractor, serializableComparator);
comparator = Comparator.comparing(keyExtractor, orderBy.getIsAscending() ? serializableComparator : reverseSerializableComparator);
}
else
{
comparator = comparator.thenComparing(keyExtractor, serializableComparator);
}
if(!orderBy.getIsAscending())
{
comparator = comparator.reversed();
comparator = comparator.thenComparing(keyExtractor, orderBy.getIsAscending() ? serializableComparator : reverseSerializableComparator);
}
}
@ -696,6 +709,57 @@ public class MemoryRecordStore
/*******************************************************************************
**
*******************************************************************************/
private Serializable applyFormatString(Serializable value, GroupBy groupBy) throws QException
{
if(value == null)
{
return (null);
}
String formatString = groupBy.getFormatString();
try
{
if(formatString.startsWith("DATE_FORMAT"))
{
/////////////////////////////////////////////////////////////////////////////
// one known-use case we have here looks like this: //
// DATE_FORMAT(CONVERT_TZ(%s, 'UTC', 'UTC'), '%%Y-%%m-%%dT%%H') //
// ... for now, let's just try to support the formatting bit at the end... //
// todo - support the CONVERT_TZ bit too! //
/////////////////////////////////////////////////////////////////////////////
String sqlDateTimeFormat = formatString.replaceFirst(".*'%%", "%%").replaceFirst("'.*", "");
DateTimeFormatter dateTimeFormatter = DateTimeGroupBy.sqlDateFormatToSelectedDateTimeFormatter(sqlDateTimeFormat);
if(dateTimeFormatter == null)
{
throw (new QException("Unsupported sql dateTime format string [" + sqlDateTimeFormat + "] for MemoryRecordStore"));
}
String valueAsString = ValueUtils.getValueAsString(value);
Instant valueAsInstant = ValueUtils.getValueAsInstant(valueAsString);
ZonedDateTime zonedDateTime = valueAsInstant.atZone(ZoneId.systemDefault());
return (dateTimeFormatter.format(zonedDateTime));
}
else
{
throw (new QException("Unsupported group-by format string [" + formatString + "] for MemoryRecordStore"));
}
}
catch(QException qe)
{
throw (qe);
}
catch(Exception e)
{
throw (new QException("Error applying format string [" + formatString + "] to group by value [" + value + "]", e));
}
}
/*******************************************************************************
**
*******************************************************************************/

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java
View File

@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
import java.io.Serializable;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.time.Instant;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@ -173,11 +175,10 @@ public class ColumnStatsStep implements BackendStep
Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
GroupBy groupBy = new GroupBy(field.getType(), fieldName);
// todo - something here about "by-date, not time"
// todo - something here about an input param to specify how you want dates & date-times grouped
if(field.getType().equals(QFieldType.DATE_TIME))
{
// groupBy = new GroupBy(field.getType(), fieldName, "DATE(%s)");
String sqlExpression = DateTimeGroupBy.HOUR.getSqlExpression();
String sqlExpression = DateTimeGroupBy.HOUR.getSqlExpression(ZoneId.systemDefault());
groupBy = new GroupBy(QFieldType.STRING, fieldName, sqlExpression);
}
@ -230,6 +231,12 @@ public class ColumnStatsStep implements BackendStep
for(AggregateResult result : aggregateOutput.getResults())
{
Serializable value = result.getGroupByValue(groupBy);
if(field.getType().equals(QFieldType.DATE_TIME) && value != null)
{
value = Instant.parse(value + ":00:00Z");
}
Integer count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate));
valueCounts.add(new QRecord().withValue(fieldName, value).withValue("count", count));
}

47
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
import java.io.Serializable;
import java.math.BigDecimal;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
@ -91,4 +92,50 @@ class ColumnStatsStepTest extends BaseTest
.hasFieldOrPropertyWithValue("percent", new BigDecimal("16.67"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testDateTimesRollupByHour() throws QException
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(List.of(
new QRecord().withValue("timestamp", Instant.parse("2024-01-31T09:59:01Z")),
new QRecord().withValue("timestamp", Instant.parse("2024-01-31T09:59:59Z")),
new QRecord().withValue("timestamp", Instant.parse("2024-01-31T10:00:00Z")),
new QRecord().withValue("timestamp", Instant.parse("2024-01-31T10:01:01Z")),
new QRecord().withValue("timestamp", Instant.parse("2024-01-31T10:59:59Z")),
new QRecord().withValue("timestamp", null)
));
new InsertAction().execute(insertInput);
RunBackendStepInput input = new RunBackendStepInput();
input.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
input.addValue("fieldName", "timestamp");
input.addValue("orderBy", "count.desc");
RunBackendStepOutput output = new RunBackendStepOutput();
new ColumnStatsStep().run(input, output);
Map<String, Serializable> values = output.getValues();
@SuppressWarnings("unchecked")
List<QRecord> valueCounts = (List<QRecord>) values.get("valueCounts");
assertThat(valueCounts.get(0).getValues())
.hasFieldOrPropertyWithValue("timestamp", Instant.parse("2024-01-31T10:00:00Z"))
.hasFieldOrPropertyWithValue("count", 3);
assertThat(valueCounts.get(1).getValues())
.hasFieldOrPropertyWithValue("timestamp", Instant.parse("2024-01-31T09:00:00Z"))
.hasFieldOrPropertyWithValue("count", 2);
assertThat(valueCounts.get(2).getValues())
.hasFieldOrPropertyWithValue("timestamp", null)
.hasFieldOrPropertyWithValue("count", 1);
}
}