QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-19 05:30:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make column stats treat "" and null the same (as a non-value or blank)

Browse Source
This commit is contained in:
darin.kelkhoff
2023-06-01 15:02:32 -05:00
parent 6455171cee
commit f5516c8122
6 changed files with 680 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java
View File

@ -22,12 +22,14 @@
package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.Month;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@ -35,6 +37,14 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
@ -52,6 +62,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
@ -59,6 +70,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ -514,6 +526,139 @@ class MemoryBackendModuleTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testAggregate() throws QException
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(List.of(
new QRecord().withValue("noOfShoes", 1).withValue("lastName", "Simpson"),
new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Simpson"),
new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Flanders"),
new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Flanders"),
new QRecord().withValue("noOfShoes", 3).withValue("lastName", "Flanders"),
new QRecord().withValue("noOfShoes", null).withValue("lastName", "Flanders")
));
new InsertAction().execute(insertInput);
{
////////////////////////////////
// do some integer aggregates //
////////////////////////////////
AggregateInput aggregateInput = new AggregateInput();
aggregateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM));
aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.COUNT));
aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.COUNT_DISTINCT));
aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.MIN));
aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.MAX));
aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.AVG));
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
assertEquals(1, aggregateOutput.getResults().size());
AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
assertEquals(10, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals(5, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.COUNT)));
assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.COUNT_DISTINCT)));
assertEquals(1, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.MIN)));
assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.MAX)));
assertEquals(new BigDecimal(2), aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.AVG)));
}
{
///////////////////////////////
// do some string aggregates //
///////////////////////////////
AggregateInput aggregateInput = new AggregateInput();
aggregateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.COUNT));
aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.COUNT_DISTINCT));
aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.MIN));
aggregateInput.withAggregate(new Aggregate("lastName", AggregateOperator.MAX));
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
assertEquals(1, aggregateOutput.getResults().size());
AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
assertEquals(6, aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.COUNT)));
assertEquals(2, aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.COUNT_DISTINCT)));
assertEquals("Flanders", aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.MIN)));
assertEquals("Simpson", aggregateResult.getAggregateValue(new Aggregate("lastName", AggregateOperator.MAX)));
assertThatThrownBy(() -> new AggregateAction().execute(aggregateInput.withAggregates(List.of(new Aggregate("lastName", AggregateOperator.SUM))))).hasStackTraceContaining("Cannot perform SUM");
assertThatThrownBy(() -> new AggregateAction().execute(aggregateInput.withAggregates(List.of(new Aggregate("lastName", AggregateOperator.AVG))))).hasStackTraceContaining("Cannot perform AVG");
}
{
////////////////////
// do a group-bys //
////////////////////
AggregateInput aggregateInput = new AggregateInput();
aggregateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
aggregateInput.withAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM));
aggregateInput.withGroupBy(new GroupBy(QFieldType.STRING, "lastName"));
{
aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM))));
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
assertEquals(2, aggregateOutput.getResults().size());
AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
aggregateResult = aggregateOutput.getResults().get(1);
assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
}
{
///////////////////////////////////////////////////////////////////////////
// with all different versions of order-by (agg or groupBy, asc or desc) //
///////////////////////////////////////////////////////////////////////////
aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByAggregate(new Aggregate("noOfShoes", AggregateOperator.SUM)).withIsAscending(false)));
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
assertEquals(2, aggregateOutput.getResults().size());
AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
aggregateResult = aggregateOutput.getResults().get(1);
assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
}
{
aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByGroupBy(new GroupBy(QFieldType.STRING, "lastName"))));
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
assertEquals(2, aggregateOutput.getResults().size());
AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
aggregateResult = aggregateOutput.getResults().get(1);
assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
}
{
aggregateInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderByGroupBy(new GroupBy(QFieldType.STRING, "lastName")).withIsAscending(false)));
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
assertEquals(2, aggregateOutput.getResults().size());
AggregateResult aggregateResult = aggregateOutput.getResults().get(0);
assertEquals(3, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Simpson", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
aggregateResult = aggregateOutput.getResults().get(1);
assertEquals(7, aggregateResult.getAggregateValue(new Aggregate("noOfShoes", AggregateOperator.SUM)));
assertEquals("Flanders", aggregateResult.getGroupByValue(new GroupBy(QFieldType.STRING, "lastName")));
}
}
}
/*******************************************************************************
**
*******************************************************************************/

61
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java Normal file
View File

@ -0,0 +1,61 @@
package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Unit test for ColumnStatsStep
*******************************************************************************/
class ColumnStatsStepTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testEmptyStringAndNullRollUpTogether() throws QException
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(List.of(
new QRecord().withValue("noOfShoes", 1).withValue("lastName", "Simpson"),
new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Simpson"),
new QRecord().withValue("noOfShoes", 2).withValue("lastName", "Simpson"),
new QRecord().withValue("noOfShoes", 2).withValue("lastName", ""), // this record and the next one -
new QRecord().withValue("noOfShoes", 3).withValue("lastName", null), // this record and the previous - should both come out as null below
new QRecord().withValue("noOfShoes", null).withValue("lastName", "Flanders")
));
new InsertAction().execute(insertInput);
RunBackendStepInput input = new RunBackendStepInput();
input.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
input.addValue("fieldName", "lastName");
input.addValue("orderBy", "count.desc");
RunBackendStepOutput output = new RunBackendStepOutput();
new ColumnStatsStep().run(input, output);
Map<String, Serializable> values = output.getValues();
@SuppressWarnings("unchecked")
List<QRecord> valueCounts = (List<QRecord>) values.get("valueCounts");
assertThat(valueCounts.get(0).getValues()).hasFieldOrPropertyWithValue("lastName", "Simpson").hasFieldOrPropertyWithValue("count", 3);
assertThat(valueCounts.get(1).getValues()).hasFieldOrPropertyWithValue("lastName", null).hasFieldOrPropertyWithValue("count", 2); // here's the assert for the "" and null record above.
assertThat(valueCounts.get(2).getValues()).hasFieldOrPropertyWithValue("lastName", "Flanders").hasFieldOrPropertyWithValue("count", 1);
}
}