From 782a07b17608ecb03144f2c8f0679bd142003a80 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 1 Apr 2024 08:54:32 -0500 Subject: [PATCH] CE-881 - Update aggregates to include dates, plus product, variance, and standard deviation --- .../reporting/GenerateReportAction.java | 74 ++++++---- .../utils/aggregates/AggregatesInterface.java | 54 ++++++- .../aggregates/BigDecimalAggregates.java | 72 +++++++++- .../utils/aggregates/InstantAggregates.java | 136 ++++++++++++++++++ .../utils/aggregates/IntegerAggregates.java | 82 ++++++++++- .../utils/aggregates/LocalDateAggregates.java | 136 ++++++++++++++++++ .../core/utils/aggregates/LongAggregates.java | 74 +++++++++- .../utils/aggregates/VarianceCalculator.java | 117 +++++++++++++++ .../core/utils/aggregates/AggregatesTest.java | 118 ++++++++++++++- 9 files changed, 828 insertions(+), 35 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/InstantAggregates.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LocalDateAggregates.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/VarianceCalculator.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java index 84da01e0..8fe2046d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java @@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.actions.reporting; import java.io.Serializable; import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -73,7 +75,9 @@ import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.aggregates.AggregatesInterface; import com.kingsrook.qqq.backend.core.utils.aggregates.BigDecimalAggregates; +import com.kingsrook.qqq.backend.core.utils.aggregates.InstantAggregates; import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates; +import com.kingsrook.qqq.backend.core.utils.aggregates.LocalDateAggregates; import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates; @@ -103,11 +107,11 @@ public class GenerateReportAction // Aggregates: (count:47;sum:10,000;max:2,000;min:15) // // salesSummaryReport > [(state:MO),(city:St.Louis)] > salePrice > (count:47;sum:10,000;max:2,000;min:15) // ///////////////////////////////////////////////////////////////////////////////////////////////////////////// - Map>>> summaryAggregates = new HashMap<>(); - Map>>> varianceAggregates = new HashMap<>(); + Map>>> summaryAggregates = new HashMap<>(); + Map>>> varianceAggregates = new HashMap<>(); - Map> totalAggregates = new HashMap<>(); - Map> varianceTotalAggregates = new HashMap<>(); + Map> totalAggregates = new HashMap<>(); + Map> varianceTotalAggregates = new HashMap<>(); private ExportStreamerInterface reportStreamer; private List dataSources; @@ -546,9 +550,9 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List records, Map>>> aggregatesMap) + private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List records, Map>>> aggregatesMap) { - Map>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>()); + Map>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>()); for(QRecord record : records) { @@ -584,9 +588,9 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map>> viewAggregates, SummaryKey key) + private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map>> viewAggregates, SummaryKey key) { - Map> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>()); + Map> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>()); addRecordToAggregatesMap(table, record, keyAggregates); } @@ -595,29 +599,45 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map> aggregatesMap) + private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map> aggregatesMap) { for(QFieldMetaData field : table.getFields().values()) { + if(StringUtils.hasContent(field.getPossibleValueSourceName())) + { + continue; + } + if(field.getType().equals(QFieldType.INTEGER)) { @SuppressWarnings("unchecked") - AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new IntegerAggregates()); + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new IntegerAggregates()); fieldAggregates.add(record.getValueInteger(field.getName())); } else if(field.getType().equals(QFieldType.LONG)) { @SuppressWarnings("unchecked") - AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new LongAggregates()); + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new LongAggregates()); fieldAggregates.add(record.getValueLong(field.getName())); } else if(field.getType().equals(QFieldType.DECIMAL)) { @SuppressWarnings("unchecked") - AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new BigDecimalAggregates()); + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new BigDecimalAggregates()); fieldAggregates.add(record.getValueBigDecimal(field.getName())); } - // todo - more types (dates, at least?) + else if(field.getType().equals(QFieldType.DATE_TIME)) + { + @SuppressWarnings("unchecked") + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new InstantAggregates()); + fieldAggregates.add(record.getValueInstant(field.getName())); + } + else if(field.getType().equals(QFieldType.DATE)) + { + @SuppressWarnings("unchecked") + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new LocalDateAggregates()); + fieldAggregates.add(record.getValueLocalDate(field.getName())); + } } } @@ -735,11 +755,11 @@ public class GenerateReportAction // create summary rows // ///////////////////////// List summaryRows = new ArrayList<>(); - for(Map.Entry>> entry : summaryAggregates.getOrDefault(view.getName(), Collections.emptyMap()).entrySet()) + for(Map.Entry>> entry : summaryAggregates.getOrDefault(view.getName(), Collections.emptyMap()).entrySet()) { - SummaryKey summaryKey = entry.getKey(); - Map> fieldAggregates = entry.getValue(); - Map summaryValues = getSummaryValuesForInterpreter(fieldAggregates); + SummaryKey summaryKey = entry.getKey(); + Map> fieldAggregates = entry.getValue(); + Map summaryValues = getSummaryValuesForInterpreter(fieldAggregates); variableInterpreter.addValueMap("pivot", summaryValues); variableInterpreter.addValueMap("summary", summaryValues); @@ -748,9 +768,9 @@ public class GenerateReportAction if(!varianceAggregates.isEmpty()) { - Map>> varianceMap = varianceAggregates.getOrDefault(view.getName(), Collections.emptyMap()); - Map> varianceSubMap = varianceMap.getOrDefault(summaryKey, Collections.emptyMap()); - Map varianceValues = getSummaryValuesForInterpreter(varianceSubMap); + Map>> varianceMap = varianceAggregates.getOrDefault(view.getName(), Collections.emptyMap()); + Map> varianceSubMap = varianceMap.getOrDefault(summaryKey, Collections.emptyMap()); + Map varianceValues = getSummaryValuesForInterpreter(varianceSubMap); variableInterpreter.addValueMap("variancePivot", varianceValues); variableInterpreter.addValueMap("variance", varianceValues); } @@ -931,18 +951,24 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private Map getSummaryValuesForInterpreter(Map> fieldAggregates) + private Map getSummaryValuesForInterpreter(Map> fieldAggregates) { Map summaryValuesForInterpreter = new HashMap<>(); - for(Map.Entry> subEntry : fieldAggregates.entrySet()) + for(Map.Entry> subEntry : fieldAggregates.entrySet()) { - String fieldName = subEntry.getKey(); - AggregatesInterface aggregates = subEntry.getValue(); + String fieldName = subEntry.getKey(); + AggregatesInterface aggregates = subEntry.getValue(); summaryValuesForInterpreter.put("sum." + fieldName, aggregates.getSum()); summaryValuesForInterpreter.put("count." + fieldName, aggregates.getCount()); + summaryValuesForInterpreter.put("count_nums." + fieldName, aggregates.getCount()); summaryValuesForInterpreter.put("min." + fieldName, aggregates.getMin()); summaryValuesForInterpreter.put("max." + fieldName, aggregates.getMax()); summaryValuesForInterpreter.put("average." + fieldName, aggregates.getAverage()); + summaryValuesForInterpreter.put("product." + fieldName, aggregates.getProduct()); + summaryValuesForInterpreter.put("var." + fieldName, aggregates.getVariance()); + summaryValuesForInterpreter.put("varp." + fieldName, aggregates.getVarP()); + summaryValuesForInterpreter.put("std_dev." + fieldName, aggregates.getStandardDeviation()); + summaryValuesForInterpreter.put("std_devp." + fieldName, aggregates.getStdDevP()); } return summaryValuesForInterpreter; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesInterface.java index 074c2469..ee6b0a59 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesInterface.java @@ -29,8 +29,12 @@ import java.math.BigDecimal; /******************************************************************************* ** Classes that support doing data aggregations (e.g., count, sum, min, max, average). ** Sub-classes should supply the type parameter. + ** + ** The AVG_T parameter describes the type used for the average getAverage method + ** which, e.g, for date types, might be a date, vs. numbers, they'd probably be + ** BigDecimal. *******************************************************************************/ -public interface AggregatesInterface +public interface AggregatesInterface { /******************************************************************************* ** @@ -60,5 +64,51 @@ public interface AggregatesInterface /******************************************************************************* ** *******************************************************************************/ - BigDecimal getAverage(); + AVG_T getAverage(); + + + /******************************************************************************* + ** + *******************************************************************************/ + default BigDecimal getProduct() + { + return (null); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default BigDecimal getVariance() + { + return (null); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default BigDecimal getVarP() + { + return (null); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default BigDecimal getStandardDeviation() + { + return (null); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default BigDecimal getStdDevP() + { + return (null); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/BigDecimalAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/BigDecimalAggregates.java index da7f1703..76a6f0d8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/BigDecimalAggregates.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/BigDecimalAggregates.java @@ -28,13 +28,16 @@ import java.math.BigDecimal; /******************************************************************************* ** BigDecimal version of data aggregator *******************************************************************************/ -public class BigDecimalAggregates implements AggregatesInterface +public class BigDecimalAggregates implements AggregatesInterface { private int count = 0; // private Integer countDistinct; private BigDecimal sum; private BigDecimal min; private BigDecimal max; + private BigDecimal product; + + private VarianceCalculator varianceCalculator = new VarianceCalculator(); @@ -59,6 +62,15 @@ public class BigDecimalAggregates implements AggregatesInterface sum = sum.add(input); } + if(product == null) + { + product = input; + } + else + { + product = product.multiply(input); + } + if(min == null || input.compareTo(min) < 0) { min = input; @@ -68,6 +80,52 @@ public class BigDecimalAggregates implements AggregatesInterface { max = input; } + + varianceCalculator.updateVariance(input); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVariance() + { + return (varianceCalculator.getVariance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVarP() + { + return (varianceCalculator.getVarP()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStandardDeviation() + { + return (varianceCalculator.getStandardDeviation()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStdDevP() + { + return (varianceCalculator.getStdDevP()); } @@ -116,6 +174,18 @@ public class BigDecimalAggregates implements AggregatesInterface + /******************************************************************************* + ** Getter for product + ** + *******************************************************************************/ + @Override + public BigDecimal getProduct() + { + return product; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/InstantAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/InstantAggregates.java new file mode 100644 index 00000000..adb1a591 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/InstantAggregates.java @@ -0,0 +1,136 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.aggregates; + + +import java.math.BigInteger; +import java.time.Instant; + + +/******************************************************************************* + ** Instant version of data aggregator + *******************************************************************************/ +public class InstantAggregates implements AggregatesInterface +{ + private int count = 0; + // private Integer countDistinct; + + private BigInteger sumMillis = BigInteger.ZERO; + + private Instant min; + private Instant max; + + + + /******************************************************************************* + ** Add a new value to this aggregate set + *******************************************************************************/ + public void add(Instant input) + { + if(input == null) + { + return; + } + + count++; + + sumMillis = sumMillis.add(new BigInteger(String.valueOf(input.toEpochMilli()))); + + if(min == null || input.compareTo(min) < 0) + { + min = input; + } + + if(max == null || input.compareTo(max) > 0) + { + max = input; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int getCount() + { + return (count); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Instant getSum() + { + ////////////////////////////////////////// + // sum of date-times doesn't make sense // + ////////////////////////////////////////// + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Instant getMin() + { + return (min); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Instant getMax() + { + return (max); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Instant getAverage() + { + if(this.count > 0) + { + BigInteger averageMillis = this.sumMillis.divide(new BigInteger(String.valueOf(count))); + if(averageMillis.compareTo(new BigInteger(String.valueOf(Long.MAX_VALUE))) < 0) + { + return (Instant.ofEpochMilli(averageMillis.longValue())); + } + } + + return (null); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/IntegerAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/IntegerAggregates.java index 292e8a01..15efecea 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/IntegerAggregates.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/IntegerAggregates.java @@ -28,13 +28,16 @@ import java.math.BigDecimal; /******************************************************************************* ** Integer version of data aggregator *******************************************************************************/ -public class IntegerAggregates implements AggregatesInterface +public class IntegerAggregates implements AggregatesInterface { - private int count = 0; + private int count = 0; // private Integer countDistinct; - private Integer sum; - private Integer min; - private Integer max; + private Integer sum; + private Integer min; + private Integer max; + private BigDecimal product; + + private VarianceCalculator varianceCalculator = new VarianceCalculator(); @@ -48,6 +51,8 @@ public class IntegerAggregates implements AggregatesInterface return; } + BigDecimal inputBD = new BigDecimal(input); + count++; if(sum == null) @@ -59,6 +64,15 @@ public class IntegerAggregates implements AggregatesInterface sum = sum + input; } + if(product == null) + { + product = inputBD; + } + else + { + product = product.multiply(inputBD); + } + if(min == null || input < min) { min = input; @@ -68,6 +82,52 @@ public class IntegerAggregates implements AggregatesInterface { max = input; } + + varianceCalculator.updateVariance(inputBD); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVariance() + { + return (varianceCalculator.getVariance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVarP() + { + return (varianceCalculator.getVarP()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStandardDeviation() + { + return (varianceCalculator.getStandardDeviation()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStdDevP() + { + return (varianceCalculator.getStdDevP()); } @@ -116,6 +176,18 @@ public class IntegerAggregates implements AggregatesInterface + /******************************************************************************* + ** Getter for product + ** + *******************************************************************************/ + @Override + public BigDecimal getProduct() + { + return product; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LocalDateAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LocalDateAggregates.java new file mode 100644 index 00000000..3c64e200 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LocalDateAggregates.java @@ -0,0 +1,136 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.aggregates; + + +import java.math.BigInteger; +import java.time.LocalDate; + + +/******************************************************************************* + ** LocalDate version of data aggregator + *******************************************************************************/ +public class LocalDateAggregates implements AggregatesInterface +{ + private int count = 0; + // private Integer countDistinct; + + private BigInteger sumMillis = BigInteger.ZERO; + + private LocalDate min; + private LocalDate max; + + + + /******************************************************************************* + ** Add a new value to this aggregate set + *******************************************************************************/ + public void add(LocalDate input) + { + if(input == null) + { + return; + } + + count++; + + sumMillis = sumMillis.add(new BigInteger(String.valueOf(input.toEpochDay()))); + + if(min == null || input.compareTo(min) < 0) + { + min = input; + } + + if(max == null || input.compareTo(max) > 0) + { + max = input; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int getCount() + { + return (count); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public LocalDate getSum() + { + ////////////////////////////////////////// + // sum of date-times doesn't make sense // + ////////////////////////////////////////// + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public LocalDate getMin() + { + return (min); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public LocalDate getMax() + { + return (max); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public LocalDate getAverage() + { + if(this.count > 0) + { + BigInteger averageEpochDay = this.sumMillis.divide(new BigInteger(String.valueOf(count))); + if(averageEpochDay.compareTo(new BigInteger(String.valueOf(Long.MAX_VALUE))) < 0) + { + return (LocalDate.ofEpochDay(averageEpochDay.longValue())); + } + } + + return (null); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java index bcf1862b..e131cda1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java @@ -28,13 +28,16 @@ import java.math.BigDecimal; /******************************************************************************* ** Long version of data aggregator *******************************************************************************/ -public class LongAggregates implements AggregatesInterface +public class LongAggregates implements AggregatesInterface { private int count = 0; // private Long countDistinct; private Long sum; private Long min; private Long max; + private BigDecimal product; + + private VarianceCalculator varianceCalculator = new VarianceCalculator(); @@ -48,6 +51,8 @@ public class LongAggregates implements AggregatesInterface return; } + BigDecimal inputBD = new BigDecimal(input); + count++; if(sum == null) @@ -59,6 +64,15 @@ public class LongAggregates implements AggregatesInterface sum = sum + input; } + if(product == null) + { + product = inputBD; + } + else + { + product = product.multiply(inputBD); + } + if(min == null || input < min) { min = input; @@ -68,6 +82,52 @@ public class LongAggregates implements AggregatesInterface { max = input; } + + varianceCalculator.updateVariance(inputBD); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVariance() + { + return (varianceCalculator.getVariance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVarP() + { + return (varianceCalculator.getVarP()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStandardDeviation() + { + return (varianceCalculator.getStandardDeviation()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStdDevP() + { + return (varianceCalculator.getStdDevP()); } @@ -116,6 +176,18 @@ public class LongAggregates implements AggregatesInterface + /******************************************************************************* + ** Getter for product + ** + *******************************************************************************/ + @Override + public BigDecimal getProduct() + { + return product; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/VarianceCalculator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/VarianceCalculator.java new file mode 100644 index 00000000..eefe04f6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/VarianceCalculator.java @@ -0,0 +1,117 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.aggregates; + + +import java.math.BigDecimal; +import java.math.RoundingMode; + + +/******************************************************************************* + ** see https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm + ** + *******************************************************************************/ +public class VarianceCalculator +{ + private int n; + private BigDecimal runningMean = BigDecimal.ZERO; + private BigDecimal m2 = BigDecimal.ZERO; + + public static int scaleForVarianceCalculations = 4; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void updateVariance(BigDecimal newInput) + { + n++; + BigDecimal delta = newInput.subtract(runningMean); + runningMean = runningMean.add(delta.divide(new BigDecimal(n), scaleForVarianceCalculations, RoundingMode.HALF_UP)); + BigDecimal delta2 = newInput.subtract(runningMean); + m2 = m2.add(delta.multiply(delta2)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public BigDecimal getVariance() + { + if(n < 2) + { + return (null); + } + + return m2.divide(new BigDecimal(n - 1), scaleForVarianceCalculations, RoundingMode.HALF_UP); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public BigDecimal getVarP() + { + if(n < 2) + { + return (null); + } + + return m2.divide(new BigDecimal(n), scaleForVarianceCalculations, RoundingMode.HALF_UP); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public BigDecimal getStandardDeviation() + { + BigDecimal variance = getVariance(); + if(variance == null) + { + return (null); + } + + return BigDecimal.valueOf(Math.sqrt(variance.doubleValue())).setScale(scaleForVarianceCalculations, RoundingMode.HALF_UP); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public BigDecimal getStdDevP() + { + BigDecimal varP = getVarP(); + if(varP == null) + { + return (null); + } + + return BigDecimal.valueOf(Math.sqrt(varP.doubleValue())).setScale(scaleForVarianceCalculations, RoundingMode.HALF_UP); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java index 86b09257..4dff56c6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java @@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.utils.aggregates; import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.Month; import com.kingsrook.qqq.backend.core.BaseTest; import org.assertj.core.data.Offset; import org.junit.jupiter.api.Test; @@ -78,6 +81,12 @@ class AggregatesTest extends BaseTest assertEquals(15, aggregates.getMax()); assertEquals(30, aggregates.getSum()); assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("10"), Offset.offset(BigDecimal.ZERO)); + + assertEquals(new BigDecimal("750"), aggregates.getProduct()); + assertEquals(new BigDecimal("25.0000"), aggregates.getVariance()); + assertEquals(new BigDecimal("5.0000"), aggregates.getStandardDeviation()); + assertThat(aggregates.getVarP()).isCloseTo(new BigDecimal("16.6667"), Offset.offset(new BigDecimal(".0001"))); + assertThat(aggregates.getStdDevP()).isCloseTo(new BigDecimal("4.0824"), Offset.offset(new BigDecimal(".0001"))); } @@ -89,6 +98,7 @@ class AggregatesTest extends BaseTest void testBigDecimal() { BigDecimalAggregates aggregates = new BigDecimalAggregates(); + aggregates.add(null); assertEquals(0, aggregates.getCount()); assertNull(aggregates.getMin()); @@ -114,13 +124,117 @@ class AggregatesTest extends BaseTest BigDecimal bd148 = new BigDecimal("14.8"); aggregates.add(bd148); - - aggregates.add(null); assertEquals(3, aggregates.getCount()); assertEquals(bd51, aggregates.getMin()); assertEquals(bd148, aggregates.getMax()); assertEquals(new BigDecimal("30.0"), aggregates.getSum()); assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("10.0"), Offset.offset(BigDecimal.ZERO)); + + assertEquals(new BigDecimal("762.348"), aggregates.getProduct()); + assertEquals(new BigDecimal("23.5300"), aggregates.getVariance()); + assertEquals(new BigDecimal("4.8508"), aggregates.getStandardDeviation()); + assertThat(aggregates.getVarP()).isCloseTo(new BigDecimal("15.6867"), Offset.offset(new BigDecimal(".0001"))); + assertThat(aggregates.getStdDevP()).isCloseTo(new BigDecimal("3.9606"), Offset.offset(new BigDecimal(".0001"))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testInstant() + { + InstantAggregates aggregates = new InstantAggregates(); + + assertEquals(0, aggregates.getCount()); + assertNull(aggregates.getMin()); + assertNull(aggregates.getMax()); + assertNull(aggregates.getSum()); + assertNull(aggregates.getAverage()); + + Instant i1970 = Instant.parse("1970-01-01T00:00:00Z"); + aggregates.add(i1970); + assertEquals(1, aggregates.getCount()); + assertEquals(i1970, aggregates.getMin()); + assertEquals(i1970, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(i1970, aggregates.getAverage()); + + Instant i1980 = Instant.parse("1980-01-01T00:00:00Z"); + aggregates.add(i1980); + assertEquals(2, aggregates.getCount()); + assertEquals(i1970, aggregates.getMin()); + assertEquals(i1980, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(Instant.parse("1975-01-01T00:00:00Z"), aggregates.getAverage()); + + Instant i1990 = Instant.parse("1990-01-01T00:00:00Z"); + aggregates.add(i1990); + assertEquals(3, aggregates.getCount()); + assertEquals(i1970, aggregates.getMin()); + assertEquals(i1990, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(Instant.parse("1980-01-01T08:00:00Z"), aggregates.getAverage()); // a leap day throws this off by 8 hours :) + + ///////////////////////////////////////////////////////////////////// + // assert we gracefully return null for these ops we don't support // + ///////////////////////////////////////////////////////////////////// + assertNull(aggregates.getProduct()); + assertNull(aggregates.getVariance()); + assertNull(aggregates.getStandardDeviation()); + assertNull(aggregates.getVarP()); + assertNull(aggregates.getStdDevP()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testLocalDate() + { + LocalDateAggregates aggregates = new LocalDateAggregates(); + + assertEquals(0, aggregates.getCount()); + assertNull(aggregates.getMin()); + assertNull(aggregates.getMax()); + assertNull(aggregates.getSum()); + assertNull(aggregates.getAverage()); + + LocalDate ld1970 = LocalDate.of(1970, Month.JANUARY, 1); + aggregates.add(ld1970); + assertEquals(1, aggregates.getCount()); + assertEquals(ld1970, aggregates.getMin()); + assertEquals(ld1970, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(ld1970, aggregates.getAverage()); + + LocalDate ld1980 = LocalDate.of(1980, Month.JANUARY, 1); + aggregates.add(ld1980); + assertEquals(2, aggregates.getCount()); + assertEquals(ld1970, aggregates.getMin()); + assertEquals(ld1980, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(LocalDate.of(1975, Month.JANUARY, 1), aggregates.getAverage()); + + LocalDate ld1990 = LocalDate.of(1990, Month.JANUARY, 1); + aggregates.add(ld1990); + assertEquals(3, aggregates.getCount()); + assertEquals(ld1970, aggregates.getMin()); + assertEquals(ld1990, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(ld1980, aggregates.getAverage()); + + ///////////////////////////////////////////////////////////////////// + // assert we gracefully return null for these ops we don't support // + ///////////////////////////////////////////////////////////////////// + assertNull(aggregates.getProduct()); + assertNull(aggregates.getVariance()); + assertNull(aggregates.getStandardDeviation()); + assertNull(aggregates.getVarP()); + assertNull(aggregates.getStdDevP()); } } \ No newline at end of file