QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #34 from Kingsrook/dev

Browse Source 
dev into sprint-30
This commit is contained in:
darin.kelkhoff
2023-08-01 18:46:48 -05:00
committed by GitHub
parent a19a516fc0 c832028961
commit d596346c44
4 changed files with 251 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java
View File

@ -29,6 +29,7 @@ import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
@ -42,18 +43,16 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
/*******************************************************************************
** Standard/re-usable post-insert customizer, for the use case where, when we
** do an insert into table "parent", we want a record automatically inserted into
** table "child", and there's a foreign key in "parent", pointed at "child"
** e.g., named: "parent.childId".
** table "child". Optionally (based on RelationshipType), there can be a foreign
** key in "parent", pointed at "child". e.g., named: "parent.childId".
**
** A similar use-case would have the foreign key in the child table - in which case,
** we could add a "Type" enum, plus abstract method to get our "Type", then logic
** to switch behavior based on type. See existing type enum, but w/ only 1 case :)
*******************************************************************************/
public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInsertCustomizer
{
public enum RelationshipType
{
PARENT_POINTS_AT_CHILD
PARENT_POINTS_AT_CHILD,
CHILD_POINTS_AT_PARENT
}
@ -68,10 +67,17 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
*******************************************************************************/
public abstract String getChildTableName();
/*******************************************************************************
**
*******************************************************************************/
public abstract String getForeignKeyFieldName();
public String getForeignKeyFieldName()
{
return (null);
}
/*******************************************************************************
**
@ -88,7 +94,7 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
{
try
{
List<QRecord> rs = new ArrayList<>();
List<QRecord> rs = records;
List<QRecord> childrenToInsert = new ArrayList<>();
QTableMetaData table = getInsertInput().getTable();
QTableMetaData childTable = getInsertInput().getInstance().getTable(getChildTableName());
@ -97,12 +103,37 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
// iterate over the inserted records, building a list child records to insert //
// for ones missing a value in the foreign key field. //
////////////////////////////////////////////////////////////////////////////////
for(QRecord record : records)
switch(getRelationshipType())
{
if(record.getValue(getForeignKeyFieldName()) == null)
case PARENT_POINTS_AT_CHILD ->
{
childrenToInsert.add(buildChildForRecord(record));
String foreignKeyFieldName = getForeignKeyFieldName();
try
{
table.getField(foreignKeyFieldName);
}
catch(Exception e)
{
throw new QRuntimeException("For RelationshipType.PARENT_POINTS_AT_CHILD, a valid foreignKeyFieldName in the parent table must be given. "
+ "[" + foreignKeyFieldName + "] is not a valid field name in table [" + table.getName() + "]");
}
for(QRecord record : records)
{
if(record.getValue(foreignKeyFieldName) == null)
{
childrenToInsert.add(buildChildForRecord(record));
}
}
}
case CHILD_POINTS_AT_PARENT ->
{
for(QRecord record : records)
{
childrenToInsert.add(buildChildForRecord(record));
}
}
default -> throw new IllegalStateException("Unexpected value: " + getRelationshipType());
}
///////////////////////////////////////////////////////////////////////////////////
@ -129,51 +160,70 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
/////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////////////////////////
// for the PARENT_POINTS_AT_CHILD relationship type:
// iterate over the original list of records again - for any that need a child (e.g., are missing //
// foreign key), set their foreign key to a newly inserted child's key, and add them to be updated. //
//////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> recordsToUpdate = new ArrayList<>();
for(QRecord record : records)
switch(getRelationshipType())
{
Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
if(record.getValue(getForeignKeyFieldName()) == null)
case PARENT_POINTS_AT_CHILD ->
{
///////////////////////////////////////////////////////////////////////////////////////////////////
// get the corresponding child record, if it has any errors, set that as a warning in the parent //
///////////////////////////////////////////////////////////////////////////////////////////////////
QRecord childRecord = insertedRecordIterator.next();
if(CollectionUtils.nullSafeHasContents(childRecord.getErrors()))
rs = new ArrayList<>();
List<QRecord> recordsToUpdate = new ArrayList<>();
for(QRecord record : records)
{
for(QStatusMessage error : childRecord.getErrors())
Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
if(record.getValue(getForeignKeyFieldName()) == null)
{
record.addWarning(new QWarningMessage("Error creating child " + childTable.getLabel() + " (" + error.toString() + ")"));
///////////////////////////////////////////////////////////////////////////////////////////////////
// get the corresponding child record, if it has any errors, set that as a warning in the parent //
///////////////////////////////////////////////////////////////////////////////////////////////////
QRecord childRecord = insertedRecordIterator.next();
if(CollectionUtils.nullSafeHasContents(childRecord.getErrors()))
{
for(QStatusMessage error : childRecord.getErrors())
{
record.addWarning(new QWarningMessage("Error creating child " + childTable.getLabel() + " (" + error.toString() + ")"));
}
rs.add(record);
continue;
}
Serializable foreignKey = childRecord.getValue(childTable.getPrimaryKeyField());
recordsToUpdate.add(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withValue(getForeignKeyFieldName(), foreignKey));
record.setValue(getForeignKeyFieldName(), foreignKey);
rs.add(record);
}
else
{
rs.add(record);
}
rs.add(record);
continue;
}
Serializable foreignKey = childRecord.getValue(childTable.getPrimaryKeyField());
recordsToUpdate.add(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withValue(getForeignKeyFieldName(), foreignKey));
record.setValue(getForeignKeyFieldName(), foreignKey);
rs.add(record);
////////////////////////////////////////////////////////////////////////////
// update the originally inserted records to reference their new children //
////////////////////////////////////////////////////////////////////////////
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(getInsertInput().getTableName());
updateInput.setRecords(recordsToUpdate);
updateInput.setTransaction(this.insertInput.getTransaction());
new UpdateAction().execute(updateInput);
}
else
case CHILD_POINTS_AT_PARENT ->
{
rs.add(record);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// todo - some version of looking at the inserted children to confirm that they were inserted, and updating the parents with warnings if they weren't //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
}
default -> throw new IllegalStateException("Unexpected value: " + getRelationshipType());
}
////////////////////////////////////////////////////////////////////////////
// update the originally inserted records to reference their new children //
////////////////////////////////////////////////////////////////////////////
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(getInsertInput().getTableName());
updateInput.setRecords(recordsToUpdate);
updateInput.setTransaction(this.insertInput.getTransaction());
new UpdateAction().execute(updateInput);
return (rs);
}
catch(RuntimeException re)
{
throw (re);
}
catch(Exception e)
{
throw new RuntimeException("Error inserting new child records for new parent records", e);

44
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java
View File

@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
import java.io.Serializable;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@ -303,14 +305,18 @@ public class ColumnStatsStep implements BackendStep
/////////////////////////////////////////////////////////////////////////////////
// just in case any of these don't fit in an integer, use decimal for them all //
/////////////////////////////////////////////////////////////////////////////////
Aggregate countNonNullAggregate = new Aggregate(fieldName, AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
Aggregate countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT).withFieldType(QFieldType.DECIMAL);
Aggregate sumAggregate = new Aggregate(fieldName, AggregateOperator.SUM).withFieldType(QFieldType.DECIMAL);
Aggregate avgAggregate = new Aggregate(fieldName, AggregateOperator.AVG).withFieldType(QFieldType.DECIMAL);
Aggregate minAggregate = new Aggregate(fieldName, AggregateOperator.MIN);
Aggregate maxAggregate = new Aggregate(fieldName, AggregateOperator.MAX);
AggregateInput statsAggregateInput = new AggregateInput();
Aggregate countTotalRowsAggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
Aggregate countNonNullAggregate = new Aggregate(fieldName, AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
Aggregate countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT).withFieldType(QFieldType.DECIMAL);
Aggregate sumAggregate = new Aggregate(fieldName, AggregateOperator.SUM).withFieldType(QFieldType.DECIMAL);
Aggregate avgAggregate = new Aggregate(fieldName, AggregateOperator.AVG).withFieldType(QFieldType.DECIMAL);
Aggregate minAggregate = new Aggregate(fieldName, AggregateOperator.MIN);
Aggregate maxAggregate = new Aggregate(fieldName, AggregateOperator.MAX);
AggregateInput statsAggregateInput = new AggregateInput();
statsAggregateInput.withAggregate(countTotalRowsAggregate);
statsAggregateInput.withAggregate(countNonNullAggregate);
if(doCountDistinct)
{
statsAggregateInput.withAggregate(countDistinctAggregate);
@ -332,6 +338,7 @@ public class ColumnStatsStep implements BackendStep
statsAggregateInput.withAggregate(maxAggregate);
}
BigDecimal totalRows = null;
if(CollectionUtils.nullSafeHasContents(statsAggregateInput.getAggregates()))
{
statsAggregateInput.setTableName(tableName);
@ -346,6 +353,8 @@ public class ColumnStatsStep implements BackendStep
{
AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0);
totalRows = ValueUtils.getValueAsBigDecimal(statsAggregateResult.getAggregateValue(countTotalRowsAggregate));
statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate));
if(doCountDistinct)
{
@ -388,6 +397,27 @@ public class ColumnStatsStep implements BackendStep
}
}
/////////////////////
// figure count%'s //
/////////////////////
if(totalRows == null)
{
totalRows = new BigDecimal(valueCounts.stream().mapToInt(r -> r.getValueInteger("count")).sum());
}
if(totalRows != null && totalRows.compareTo(BigDecimal.ZERO) > 0)
{
BigDecimal oneHundred = new BigDecimal(100);
for(QRecord valueCount : valueCounts)
{
BigDecimal percent = new BigDecimal(Objects.requireNonNullElse(valueCount.getValueInteger("count"), 0)).divide(totalRows, 4, RoundingMode.HALF_UP).multiply(oneHundred).setScale(2, RoundingMode.HALF_UP);
valueCount.setValue("percent", percent);
}
QFieldMetaData percentField = new QFieldMetaData("percent", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.PERCENT_POINT2).withLabel("Percent");
QValueFormatter.setDisplayValuesInRecords(Map.of(fieldName, field, "percent", percentField), valueCounts);
}
QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(null);
fields.forEach(qInstanceEnricher::enrichField);

96
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizerTest.java
View File

@ -70,7 +70,7 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest
void testEmptyCases() throws QException
{
QInstance qInstance = QContext.getQInstance();
addPostInsertActionToTable(qInstance);
addPostInsertActionToPersonTable(qInstance);
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
@ -95,10 +95,21 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
private static void addPostInsertActionToTable(QInstance qInstance)
private static void addPostInsertActionToPersonTable(QInstance qInstance)
{
qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
.withCustomizer(TableCustomizers.POST_INSERT_RECORD.getRole(), new QCodeReference(PersonPostInsertAddFavoriteShapeCustomizer.class));
.withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(PersonPostInsertAddFavoriteShapeCustomizer.class));
}
/*******************************************************************************
**
*******************************************************************************/
private static void addPostInsertActionToShapeTable(QInstance qInstance)
{
qInstance.getTable(TestUtils.TABLE_NAME_SHAPE)
.withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(ShapePostInsertAddPersonCustomizer.class));
}
@ -107,10 +118,10 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest
**
*******************************************************************************/
@Test
void testSimpleCase() throws QException
void testSimpleParentPointsAtChildCase() throws QException
{
QInstance qInstance = QContext.getQInstance();
addPostInsertActionToTable(qInstance);
addPostInsertActionToPersonTable(qInstance);
assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
@ -135,10 +146,10 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest
**
*******************************************************************************/
@Test
void testComplexCase() throws QException
void testComplexParentPointsAtChildCase() throws QException
{
QInstance qInstance = QContext.getQInstance();
addPostInsertActionToTable(qInstance);
addPostInsertActionToPersonTable(qInstance);
assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
@ -169,6 +180,34 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSimpleChildPointsAtParentCase() throws QException
{
QInstance qInstance = QContext.getQInstance();
addPostInsertActionToShapeTable(qInstance);
assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size());
assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_SHAPE);
insertInput.setRecords(List.of(
new QRecord().withValue("name", "Circle")
));
InsertOutput insertOutput = new InsertAction().execute(insertInput);
Integer shapeId = insertOutput.getRecords().get(0).getValueInteger("id");
List<QRecord> personRecords = TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY);
assertEquals(1, personRecords.size());
assertEquals(shapeId, personRecords.get(0).getValue("favoriteShapeId"));
assertEquals("loves Circle", personRecords.get(0).getValue("lastName"));
}
/*******************************************************************************
** for the person table - where we do PARENT_POINTS_AT_CHILD
*******************************************************************************/
public static class PersonPostInsertAddFavoriteShapeCustomizer extends ChildInserterPostInsertCustomizer
{
@ -215,4 +254,47 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest
}
}
/*******************************************************************************
** for the shape table - where we do CHILD_POINTS_AT_PARENT
*******************************************************************************/
public static class ShapePostInsertAddPersonCustomizer extends ChildInserterPostInsertCustomizer
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public QRecord buildChildForRecord(QRecord parentRecord) throws QException
{
return (new QRecord()
.withValue("firstName", "Someone who")
.withValue("lastName", "loves " + parentRecord.getValue("name"))
.withValue("favoriteShapeId", parentRecord.getValue("id")));
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String getChildTableName()
{
return (TestUtils.TABLE_NAME_PERSON_MEMORY);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public RelationshipType getRelationshipType()
{
return (RelationshipType.CHILD_POINTS_AT_PARENT);
}
}
}

39
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java
View File

@ -1,7 +1,29 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
import java.io.Serializable;
import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
@ -53,9 +75,20 @@ class ColumnStatsStepTest extends BaseTest
@SuppressWarnings("unchecked")
List<QRecord> valueCounts = (List<QRecord>) values.get("valueCounts");
assertThat(valueCounts.get(0).getValues()).hasFieldOrPropertyWithValue("lastName", "Simpson").hasFieldOrPropertyWithValue("count", 3);
assertThat(valueCounts.get(1).getValues()).hasFieldOrPropertyWithValue("lastName", null).hasFieldOrPropertyWithValue("count", 2); // here's the assert for the "" and null record above.
assertThat(valueCounts.get(2).getValues()).hasFieldOrPropertyWithValue("lastName", "Flanders").hasFieldOrPropertyWithValue("count", 1);
assertThat(valueCounts.get(0).getValues())
.hasFieldOrPropertyWithValue("lastName", "Simpson")
.hasFieldOrPropertyWithValue("count", 3)
.hasFieldOrPropertyWithValue("percent", new BigDecimal("50.00"));
assertThat(valueCounts.get(1).getValues())
.hasFieldOrPropertyWithValue("lastName", null)
.hasFieldOrPropertyWithValue("count", 2) // here's the assert for the "" and null record above.
.hasFieldOrPropertyWithValue("percent", new BigDecimal("33.33"));
assertThat(valueCounts.get(2).getValues())
.hasFieldOrPropertyWithValue("lastName", "Flanders")
.hasFieldOrPropertyWithValue("count", 1)
.hasFieldOrPropertyWithValue("percent", new BigDecimal("16.67"));
}
}