From 774309e846a2bead64fb3ea450d98eb2f14c60a3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 27 Jul 2023 08:37:05 -0500 Subject: [PATCH 1/2] Add percents to ColumnStats --- .../columnstats/ColumnStatsStep.java | 44 ++++++++++++++++--- .../columnstats/ColumnStatsStepTest.java | 39 ++++++++++++++-- 2 files changed, 73 insertions(+), 10 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java index ae5a12ed..53816fae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java @@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats; import java.io.Serializable; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -303,14 +305,18 @@ public class ColumnStatsStep implements BackendStep ///////////////////////////////////////////////////////////////////////////////// // just in case any of these don't fit in an integer, use decimal for them all // ///////////////////////////////////////////////////////////////////////////////// - Aggregate countNonNullAggregate = new Aggregate(fieldName, AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL); - Aggregate countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT).withFieldType(QFieldType.DECIMAL); - Aggregate sumAggregate = new Aggregate(fieldName, AggregateOperator.SUM).withFieldType(QFieldType.DECIMAL); - Aggregate avgAggregate = new Aggregate(fieldName, AggregateOperator.AVG).withFieldType(QFieldType.DECIMAL); - Aggregate minAggregate = new Aggregate(fieldName, AggregateOperator.MIN); - Aggregate maxAggregate = new Aggregate(fieldName, AggregateOperator.MAX); - AggregateInput statsAggregateInput = new AggregateInput(); + Aggregate countTotalRowsAggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL); + Aggregate countNonNullAggregate = new Aggregate(fieldName, AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL); + Aggregate countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT).withFieldType(QFieldType.DECIMAL); + Aggregate sumAggregate = new Aggregate(fieldName, AggregateOperator.SUM).withFieldType(QFieldType.DECIMAL); + Aggregate avgAggregate = new Aggregate(fieldName, AggregateOperator.AVG).withFieldType(QFieldType.DECIMAL); + Aggregate minAggregate = new Aggregate(fieldName, AggregateOperator.MIN); + Aggregate maxAggregate = new Aggregate(fieldName, AggregateOperator.MAX); + + AggregateInput statsAggregateInput = new AggregateInput(); + statsAggregateInput.withAggregate(countTotalRowsAggregate); statsAggregateInput.withAggregate(countNonNullAggregate); + if(doCountDistinct) { statsAggregateInput.withAggregate(countDistinctAggregate); @@ -332,6 +338,7 @@ public class ColumnStatsStep implements BackendStep statsAggregateInput.withAggregate(maxAggregate); } + BigDecimal totalRows = null; if(CollectionUtils.nullSafeHasContents(statsAggregateInput.getAggregates())) { statsAggregateInput.setTableName(tableName); @@ -346,6 +353,8 @@ public class ColumnStatsStep implements BackendStep { AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0); + totalRows = ValueUtils.getValueAsBigDecimal(statsAggregateResult.getAggregateValue(countTotalRowsAggregate)); + statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate)); if(doCountDistinct) { @@ -388,6 +397,27 @@ public class ColumnStatsStep implements BackendStep } } + ///////////////////// + // figure count%'s // + ///////////////////// + if(totalRows == null) + { + totalRows = new BigDecimal(valueCounts.stream().mapToInt(r -> r.getValueInteger("count")).sum()); + } + + if(totalRows != null && totalRows.compareTo(BigDecimal.ZERO) > 0) + { + BigDecimal oneHundred = new BigDecimal(100); + for(QRecord valueCount : valueCounts) + { + BigDecimal percent = new BigDecimal(Objects.requireNonNullElse(valueCount.getValueInteger("count"), 0)).divide(totalRows, 4, RoundingMode.HALF_UP).multiply(oneHundred).setScale(2, RoundingMode.HALF_UP); + valueCount.setValue("percent", percent); + } + + QFieldMetaData percentField = new QFieldMetaData("percent", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.PERCENT_POINT2).withLabel("Percent"); + QValueFormatter.setDisplayValuesInRecords(Map.of(fieldName, field, "percent", percentField), valueCounts); + } + QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(null); fields.forEach(qInstanceEnricher::enrichField); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java index 0e7f9ed9..d2b6fa0b 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java @@ -1,7 +1,29 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.processes.implementations.columnstats; import java.io.Serializable; +import java.math.BigDecimal; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; @@ -53,9 +75,20 @@ class ColumnStatsStepTest extends BaseTest @SuppressWarnings("unchecked") List valueCounts = (List) values.get("valueCounts"); - assertThat(valueCounts.get(0).getValues()).hasFieldOrPropertyWithValue("lastName", "Simpson").hasFieldOrPropertyWithValue("count", 3); - assertThat(valueCounts.get(1).getValues()).hasFieldOrPropertyWithValue("lastName", null).hasFieldOrPropertyWithValue("count", 2); // here's the assert for the "" and null record above. - assertThat(valueCounts.get(2).getValues()).hasFieldOrPropertyWithValue("lastName", "Flanders").hasFieldOrPropertyWithValue("count", 1); + assertThat(valueCounts.get(0).getValues()) + .hasFieldOrPropertyWithValue("lastName", "Simpson") + .hasFieldOrPropertyWithValue("count", 3) + .hasFieldOrPropertyWithValue("percent", new BigDecimal("50.00")); + + assertThat(valueCounts.get(1).getValues()) + .hasFieldOrPropertyWithValue("lastName", null) + .hasFieldOrPropertyWithValue("count", 2) // here's the assert for the "" and null record above. + .hasFieldOrPropertyWithValue("percent", new BigDecimal("33.33")); + + assertThat(valueCounts.get(2).getValues()) + .hasFieldOrPropertyWithValue("lastName", "Flanders") + .hasFieldOrPropertyWithValue("count", 1) + .hasFieldOrPropertyWithValue("percent", new BigDecimal("16.67")); } } \ No newline at end of file From c832028961a381736b265f268bb0230c13945658 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 1 Aug 2023 08:57:24 -0500 Subject: [PATCH 2/2] Implement CHILD_POINTS_AT_PARENT use-case --- .../ChildInserterPostInsertCustomizer.java | 128 ++++++++++++------ ...ChildInserterPostInsertCustomizerTest.java | 96 ++++++++++++- 2 files changed, 178 insertions(+), 46 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java index bb370aa0..84363343 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java @@ -29,6 +29,7 @@ import java.util.List; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; @@ -42,18 +43,16 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; /******************************************************************************* ** Standard/re-usable post-insert customizer, for the use case where, when we ** do an insert into table "parent", we want a record automatically inserted into - ** table "child", and there's a foreign key in "parent", pointed at "child" - ** e.g., named: "parent.childId". + ** table "child". Optionally (based on RelationshipType), there can be a foreign + ** key in "parent", pointed at "child". e.g., named: "parent.childId". ** - ** A similar use-case would have the foreign key in the child table - in which case, - ** we could add a "Type" enum, plus abstract method to get our "Type", then logic - ** to switch behavior based on type. See existing type enum, but w/ only 1 case :) *******************************************************************************/ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInsertCustomizer { public enum RelationshipType { - PARENT_POINTS_AT_CHILD + PARENT_POINTS_AT_CHILD, + CHILD_POINTS_AT_PARENT } @@ -68,10 +67,17 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse *******************************************************************************/ public abstract String getChildTableName(); + + /******************************************************************************* ** *******************************************************************************/ - public abstract String getForeignKeyFieldName(); + public String getForeignKeyFieldName() + { + return (null); + } + + /******************************************************************************* ** @@ -88,7 +94,7 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse { try { - List rs = new ArrayList<>(); + List rs = records; List childrenToInsert = new ArrayList<>(); QTableMetaData table = getInsertInput().getTable(); QTableMetaData childTable = getInsertInput().getInstance().getTable(getChildTableName()); @@ -97,12 +103,37 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse // iterate over the inserted records, building a list child records to insert // // for ones missing a value in the foreign key field. // //////////////////////////////////////////////////////////////////////////////// - for(QRecord record : records) + switch(getRelationshipType()) { - if(record.getValue(getForeignKeyFieldName()) == null) + case PARENT_POINTS_AT_CHILD -> { - childrenToInsert.add(buildChildForRecord(record)); + String foreignKeyFieldName = getForeignKeyFieldName(); + try + { + table.getField(foreignKeyFieldName); + } + catch(Exception e) + { + throw new QRuntimeException("For RelationshipType.PARENT_POINTS_AT_CHILD, a valid foreignKeyFieldName in the parent table must be given. " + + "[" + foreignKeyFieldName + "] is not a valid field name in table [" + table.getName() + "]"); + } + + for(QRecord record : records) + { + if(record.getValue(foreignKeyFieldName) == null) + { + childrenToInsert.add(buildChildForRecord(record)); + } + } } + case CHILD_POINTS_AT_PARENT -> + { + for(QRecord record : records) + { + childrenToInsert.add(buildChildForRecord(record)); + } + } + default -> throw new IllegalStateException("Unexpected value: " + getRelationshipType()); } /////////////////////////////////////////////////////////////////////////////////// @@ -129,51 +160,70 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse ///////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////// + // for the PARENT_POINTS_AT_CHILD relationship type: // iterate over the original list of records again - for any that need a child (e.g., are missing // // foreign key), set their foreign key to a newly inserted child's key, and add them to be updated. // ////////////////////////////////////////////////////////////////////////////////////////////////////// - List recordsToUpdate = new ArrayList<>(); - for(QRecord record : records) + switch(getRelationshipType()) { - Serializable primaryKey = record.getValue(table.getPrimaryKeyField()); - if(record.getValue(getForeignKeyFieldName()) == null) + case PARENT_POINTS_AT_CHILD -> { - /////////////////////////////////////////////////////////////////////////////////////////////////// - // get the corresponding child record, if it has any errors, set that as a warning in the parent // - /////////////////////////////////////////////////////////////////////////////////////////////////// - QRecord childRecord = insertedRecordIterator.next(); - if(CollectionUtils.nullSafeHasContents(childRecord.getErrors())) + rs = new ArrayList<>(); + List recordsToUpdate = new ArrayList<>(); + for(QRecord record : records) { - for(QStatusMessage error : childRecord.getErrors()) + Serializable primaryKey = record.getValue(table.getPrimaryKeyField()); + if(record.getValue(getForeignKeyFieldName()) == null) { - record.addWarning(new QWarningMessage("Error creating child " + childTable.getLabel() + " (" + error.toString() + ")")); + /////////////////////////////////////////////////////////////////////////////////////////////////// + // get the corresponding child record, if it has any errors, set that as a warning in the parent // + /////////////////////////////////////////////////////////////////////////////////////////////////// + QRecord childRecord = insertedRecordIterator.next(); + if(CollectionUtils.nullSafeHasContents(childRecord.getErrors())) + { + for(QStatusMessage error : childRecord.getErrors()) + { + record.addWarning(new QWarningMessage("Error creating child " + childTable.getLabel() + " (" + error.toString() + ")")); + } + rs.add(record); + continue; + } + + Serializable foreignKey = childRecord.getValue(childTable.getPrimaryKeyField()); + recordsToUpdate.add(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withValue(getForeignKeyFieldName(), foreignKey)); + record.setValue(getForeignKeyFieldName(), foreignKey); + rs.add(record); + } + else + { + rs.add(record); } - rs.add(record); - continue; } - Serializable foreignKey = childRecord.getValue(childTable.getPrimaryKeyField()); - recordsToUpdate.add(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withValue(getForeignKeyFieldName(), foreignKey)); - record.setValue(getForeignKeyFieldName(), foreignKey); - rs.add(record); + //////////////////////////////////////////////////////////////////////////// + // update the originally inserted records to reference their new children // + //////////////////////////////////////////////////////////////////////////// + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(getInsertInput().getTableName()); + updateInput.setRecords(recordsToUpdate); + updateInput.setTransaction(this.insertInput.getTransaction()); + new UpdateAction().execute(updateInput); } - else + case CHILD_POINTS_AT_PARENT -> { - rs.add(record); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // todo - some version of looking at the inserted children to confirm that they were inserted, and updating the parents with warnings if they weren't // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// } + default -> throw new IllegalStateException("Unexpected value: " + getRelationshipType()); } - //////////////////////////////////////////////////////////////////////////// - // update the originally inserted records to reference their new children // - //////////////////////////////////////////////////////////////////////////// - UpdateInput updateInput = new UpdateInput(); - updateInput.setTableName(getInsertInput().getTableName()); - updateInput.setRecords(recordsToUpdate); - updateInput.setTransaction(this.insertInput.getTransaction()); - new UpdateAction().execute(updateInput); - return (rs); } + catch(RuntimeException re) + { + throw (re); + } catch(Exception e) { throw new RuntimeException("Error inserting new child records for new parent records", e); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizerTest.java index 13a92d5b..6032c7f9 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizerTest.java @@ -70,7 +70,7 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest void testEmptyCases() throws QException { QInstance qInstance = QContext.getQInstance(); - addPostInsertActionToTable(qInstance); + addPostInsertActionToPersonTable(qInstance); InsertInput insertInput = new InsertInput(); insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); @@ -95,10 +95,21 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ - private static void addPostInsertActionToTable(QInstance qInstance) + private static void addPostInsertActionToPersonTable(QInstance qInstance) { qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) - .withCustomizer(TableCustomizers.POST_INSERT_RECORD.getRole(), new QCodeReference(PersonPostInsertAddFavoriteShapeCustomizer.class)); + .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(PersonPostInsertAddFavoriteShapeCustomizer.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void addPostInsertActionToShapeTable(QInstance qInstance) + { + qInstance.getTable(TestUtils.TABLE_NAME_SHAPE) + .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(ShapePostInsertAddPersonCustomizer.class)); } @@ -107,10 +118,10 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest ** *******************************************************************************/ @Test - void testSimpleCase() throws QException + void testSimpleParentPointsAtChildCase() throws QException { QInstance qInstance = QContext.getQInstance(); - addPostInsertActionToTable(qInstance); + addPostInsertActionToPersonTable(qInstance); assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size()); @@ -135,10 +146,10 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest ** *******************************************************************************/ @Test - void testComplexCase() throws QException + void testComplexParentPointsAtChildCase() throws QException { QInstance qInstance = QContext.getQInstance(); - addPostInsertActionToTable(qInstance); + addPostInsertActionToPersonTable(qInstance); assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size()); @@ -169,6 +180,34 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ + @Test + void testSimpleChildPointsAtParentCase() throws QException + { + QInstance qInstance = QContext.getQInstance(); + addPostInsertActionToShapeTable(qInstance); + + assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size()); + assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size()); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_SHAPE); + insertInput.setRecords(List.of( + new QRecord().withValue("name", "Circle") + )); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + Integer shapeId = insertOutput.getRecords().get(0).getValueInteger("id"); + + List personRecords = TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY); + assertEquals(1, personRecords.size()); + assertEquals(shapeId, personRecords.get(0).getValue("favoriteShapeId")); + assertEquals("loves Circle", personRecords.get(0).getValue("lastName")); + } + + + + /******************************************************************************* + ** for the person table - where we do PARENT_POINTS_AT_CHILD + *******************************************************************************/ public static class PersonPostInsertAddFavoriteShapeCustomizer extends ChildInserterPostInsertCustomizer { @@ -215,4 +254,47 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest } } + + + /******************************************************************************* + ** for the shape table - where we do CHILD_POINTS_AT_PARENT + *******************************************************************************/ + public static class ShapePostInsertAddPersonCustomizer extends ChildInserterPostInsertCustomizer + { + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QRecord buildChildForRecord(QRecord parentRecord) throws QException + { + return (new QRecord() + .withValue("firstName", "Someone who") + .withValue("lastName", "loves " + parentRecord.getValue("name")) + .withValue("favoriteShapeId", parentRecord.getValue("id"))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getChildTableName() + { + return (TestUtils.TABLE_NAME_PERSON_MEMORY); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public RelationshipType getRelationshipType() + { + return (RelationshipType.CHILD_POINTS_AT_PARENT); + } + } + } \ No newline at end of file