QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move skip & limit out of QueryInput, into QQueryFilter...

Browse Source
This commit is contained in:
darin.kelkhoff
2023-04-26 10:18:44 -05:00
parent caf9f102f6
commit 04a8fa94f9
21 changed files with 244 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java
View File

@ -194,13 +194,13 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(record.getValue(joinOn.getLeftField()))));
}
filter.setOrderBys(join.getOrderBys());
filter.setLimit(maxRows);
QueryInput queryInput = new QueryInput();
queryInput.setTableName(join.getRightTable());
queryInput.setShouldTranslatePossibleValues(true);
queryInput.setShouldGenerateDisplayValues(true);
queryInput.setFilter(filter);
queryInput.setLimit(maxRows);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
QTableMetaData table = input.getInstance().getTable(join.getRightTable());

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java
View File

@ -156,8 +156,7 @@ public class StoreAssociatedScriptAction
queryInput.setFilter(new QQueryFilter()
.withCriteria(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, List.of(script.getValue("id"))))
.withOrderBy(new QFilterOrderBy("sequenceNo", false))
);
queryInput.setLimit(1);
.withLimit(1));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
if(!queryOutput.getRecords().isEmpty())
{

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java
View File

@ -247,7 +247,7 @@ public class SearchPossibleValueSourceAction
queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
// todo - skip & limit as params
queryInput.setLimit(250);
queryFilter.setLimit(250);
///////////////////////////////////////////////////////////////////////////////////////////////////////
// if given a default filter, make it the 'top level' filter and the one we just created a subfilter //

69
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java
View File

@ -47,6 +47,13 @@ public class QQueryFilter implements Serializable, Cloneable
private BooleanOperator booleanOperator = BooleanOperator.AND;
private List<QQueryFilter> subFilters = new ArrayList<>();
////////////////////////////////////////////////////////////////////////////////////////////////////////////
// skip & limit are meant to only apply to QueryAction (at least at the initial time they are added here) //
// e.g., they are ignored in CountAction, AggregateAction, etc, where their meanings may be less obvious //
////////////////////////////////////////////////////////////////////////////////////////////////////////////
private Integer skip;
private Integer limit;
/*******************************************************************************
@ -398,4 +405,66 @@ public class QQueryFilter implements Serializable, Cloneable
}
}
/*******************************************************************************
** Getter for skip
*******************************************************************************/
public Integer getSkip()
{
return (this.skip);
}
/*******************************************************************************
** Setter for skip
*******************************************************************************/
public void setSkip(Integer skip)
{
this.skip = skip;
}
/*******************************************************************************
** Fluent setter for skip
*******************************************************************************/
public QQueryFilter withSkip(Integer skip)
{
this.skip = skip;
return (this);
}
/*******************************************************************************
** Getter for limit
*******************************************************************************/
public Integer getLimit()
{
return (this.limit);
}
/*******************************************************************************
** Setter for limit
*******************************************************************************/
public void setLimit(Integer limit)
{
this.limit = limit;
}
/*******************************************************************************
** Fluent setter for limit
*******************************************************************************/
public QQueryFilter withLimit(Integer limit)
{
this.limit = limit;
return (this);
}
}

102
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java
View File

@ -39,8 +39,6 @@ public class QueryInput extends AbstractTableActionInput
{
private QBackendTransaction transaction;
private QQueryFilter filter;
private Integer skip;
private Integer limit;
private RecordPipe recordPipe;
@ -55,7 +53,8 @@ public class QueryInput extends AbstractTableActionInput
/////////////////////////////////////////////////////////////////////////////////////////
private Set<String> fieldsToTranslatePossibleValues;
private List<QueryJoin> queryJoins = null;
private List<QueryJoin> queryJoins = null;
private boolean selectDistinct = false;
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if you say you want to includeAssociations, you can limit which ones by passing them in associationNamesToInclude. //
@ -98,50 +97,6 @@ public class QueryInput extends AbstractTableActionInput
/*******************************************************************************
** Getter for skip
**
*******************************************************************************/
public Integer getSkip()
{
return skip;
}
/*******************************************************************************
** Setter for skip
**
*******************************************************************************/
public void setSkip(Integer skip)
{
this.skip = skip;
}
/*******************************************************************************
** Getter for limit
**
*******************************************************************************/
public Integer getLimit()
{
return limit;
}
/*******************************************************************************
** Setter for limit
**
*******************************************************************************/
public void setLimit(Integer limit)
{
this.limit = limit;
}
/*******************************************************************************
** Getter for recordPipe
**
@ -359,28 +314,6 @@ public class QueryInput extends AbstractTableActionInput
/*******************************************************************************
** Fluent setter for skip
*******************************************************************************/
public QueryInput withSkip(Integer skip)
{
this.skip = skip;
return (this);
}
/*******************************************************************************
** Fluent setter for limit
*******************************************************************************/
public QueryInput withLimit(Integer limit)
{
this.limit = limit;
return (this);
}
/*******************************************************************************
** Fluent setter for recordPipe
*******************************************************************************/
@ -497,4 +430,35 @@ public class QueryInput extends AbstractTableActionInput
return (this);
}
/*******************************************************************************
** Getter for selectDistinct
*******************************************************************************/
public boolean getSelectDistinct()
{
return (this.selectDistinct);
}
/*******************************************************************************
** Setter for selectDistinct
*******************************************************************************/
public void setSelectDistinct(boolean selectDistinct)
{
this.selectDistinct = selectDistinct;
}
/*******************************************************************************
** Fluent setter for selectDistinct
*******************************************************************************/
public QueryInput withSelectDistinct(boolean selectDistinct)
{
this.selectDistinct = selectDistinct;
return (this);
}
}

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/WidgetQueryField.java
View File

@ -62,8 +62,9 @@ public class WidgetQueryField extends AbstractWidgetValueSourceWithFilter
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(tableName);
queryInput.setFilter(getEffectiveFilter(input));
queryInput.setLimit(1);
QQueryFilter effectiveFilter = getEffectiveFilter(input);
queryInput.setFilter(effectiveFilter);
effectiveFilter.setLimit(1);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))
{

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryAction.java
View File

@ -69,7 +69,7 @@ public class EnumerationQueryAction implements QueryInterface
}
BackendQueryFilterUtils.sortRecordList(queryInput.getFilter(), recordList);
recordList = BackendQueryFilterUtils.applySkipAndLimit(queryInput, recordList);
recordList = BackendQueryFilterUtils.applySkipAndLimit(queryInput.getFilter(), recordList);
QueryOutput queryOutput = new QueryOutput(queryInput);
queryOutput.addRecords(recordList);

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java
View File

@ -27,7 +27,6 @@ import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.Month;
import java.util.Objects;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -36,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
/*******************************************************************************
@ -59,7 +59,8 @@ public class MockQueryAction implements QueryInterface
QueryOutput queryOutput = new QueryOutput(queryInput);
int rows = Objects.requireNonNullElse(queryInput.getLimit(), 1);
@SuppressWarnings("UnnecessaryUnboxing") // force an un-boxing, to force an NPE if it's null, to get to the "else 1"
int rows = ObjectUtils.tryElse(() -> queryInput.getFilter().getLimit().intValue(), 1);
for(int i = 0; i < rows; i++)
{
QRecord record = new QRecord();

82
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java
View File

@ -33,7 +33,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -129,42 +128,42 @@ public class BackendQueryFilterUtils
public static boolean doesCriteriaMatch(QFilterCriteria criterion, String fieldName, Serializable value)
{
boolean criterionMatches = switch(criterion.getOperator())
{
case EQUALS -> testEquals(criterion, value);
case NOT_EQUALS -> !testEquals(criterion, value);
case IN -> testIn(criterion, value);
case NOT_IN -> !testIn(criterion, value);
case IS_BLANK -> testBlank(criterion, value);
case IS_NOT_BLANK -> !testBlank(criterion, value);
case CONTAINS -> testContains(criterion, fieldName, value);
case NOT_CONTAINS -> !testContains(criterion, fieldName, value);
case IS_NULL_OR_IN -> testBlank(criterion, value) || testIn(criterion, value);
case LIKE -> testLike(criterion, fieldName, value);
case NOT_LIKE -> !testLike(criterion, fieldName, value);
case STARTS_WITH -> testStartsWith(criterion, fieldName, value);
case NOT_STARTS_WITH -> !testStartsWith(criterion, fieldName, value);
case ENDS_WITH -> testEndsWith(criterion, fieldName, value);
case NOT_ENDS_WITH -> !testEndsWith(criterion, fieldName, value);
case GREATER_THAN -> testGreaterThan(criterion, value);
case GREATER_THAN_OR_EQUALS -> testGreaterThan(criterion, value) || testEquals(criterion, value);
case LESS_THAN -> !testGreaterThan(criterion, value) && !testEquals(criterion, value);
case LESS_THAN_OR_EQUALS -> !testGreaterThan(criterion, value);
case BETWEEN ->
{
case EQUALS -> testEquals(criterion, value);
case NOT_EQUALS -> !testEquals(criterion, value);
case IN -> testIn(criterion, value);
case NOT_IN -> !testIn(criterion, value);
case IS_BLANK -> testBlank(criterion, value);
case IS_NOT_BLANK -> !testBlank(criterion, value);
case CONTAINS -> testContains(criterion, fieldName, value);
case NOT_CONTAINS -> !testContains(criterion, fieldName, value);
case IS_NULL_OR_IN -> testBlank(criterion, value) || testIn(criterion, value);
case LIKE -> testLike(criterion, fieldName, value);
case NOT_LIKE -> !testLike(criterion, fieldName, value);
case STARTS_WITH -> testStartsWith(criterion, fieldName, value);
case NOT_STARTS_WITH -> !testStartsWith(criterion, fieldName, value);
case ENDS_WITH -> testEndsWith(criterion, fieldName, value);
case NOT_ENDS_WITH -> !testEndsWith(criterion, fieldName, value);
case GREATER_THAN -> testGreaterThan(criterion, value);
case GREATER_THAN_OR_EQUALS -> testGreaterThan(criterion, value) || testEquals(criterion, value);
case LESS_THAN -> !testGreaterThan(criterion, value) && !testEquals(criterion, value);
case LESS_THAN_OR_EQUALS -> !testGreaterThan(criterion, value);
case BETWEEN ->
{
QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues());
QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues()));
criteria1.getValues().remove(0);
yield (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value));
}
case NOT_BETWEEN ->
{
QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues());
QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues()));
criteria1.getValues().remove(0);
boolean between = (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value));
yield !between;
}
};
QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues());
QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues()));
criteria1.getValues().remove(0);
yield (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value));
}
case NOT_BETWEEN ->
{
QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues());
QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues()));
criteria1.getValues().remove(0);
boolean between = (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value));
yield !between;
}
};
return criterionMatches;
}
@ -524,9 +523,14 @@ public class BackendQueryFilterUtils
/*******************************************************************************
** Apply skip & limit attributes from queryInput to a list of records.
*******************************************************************************/
public static List<QRecord> applySkipAndLimit(QueryInput queryInput, List<QRecord> recordList)
public static List<QRecord> applySkipAndLimit(QQueryFilter queryFilter, List<QRecord> recordList)
{
Integer skip = queryInput.getSkip();
if(queryFilter == null)
{
return (recordList);
}
Integer skip = queryFilter.getSkip();
if(skip != null && skip > 0)
{
if(skip < recordList.size())
@ -539,7 +543,7 @@ public class BackendQueryFilterUtils
}
}
Integer limit = queryInput.getLimit();
Integer limit = queryFilter.getLimit();
if(limit != null && limit >= 0 && limit < recordList.size())
{
recordList = recordList.subList(0, limit);

43
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java
View File

@ -29,6 +29,7 @@ import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
@ -52,6 +53,8 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
*******************************************************************************/
public class ExtractViaQueryStep extends AbstractExtractStep
{
private static final QLogger LOG = QLogger.getLogger(ExtractViaQueryStep.class);
public static final String FIELD_SOURCE_TABLE = "sourceTable";
private QQueryFilter queryFilter;
@ -77,11 +80,33 @@ public class ExtractViaQueryStep extends AbstractExtractStep
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
//////////////////////////////////////////////////////////////////
// clone the filter, since we're going to edit it (set a limit) //
//////////////////////////////////////////////////////////////////
QQueryFilter filterClone = queryFilter.clone();
//////////////////////////////////////////////////////////////////////////////////////////////
// if there's a limit in the extract step (e.g., the 20-record limit on the preview screen) //
// then set that limit in the filter - UNLESS - there's already a limit in the filter for //
// a smaller number of records. //
//////////////////////////////////////////////////////////////////////////////////////////////
if(getLimit() != null)
{
if(filterClone.getLimit() != null && filterClone.getLimit() < getLimit())
{
LOG.trace("Using filter's limit [" + filterClone.getLimit() + "] rather than step's limit [" + getLimit() + "]");
}
else
{
filterClone.setLimit(getLimit());
}
}
QueryInput queryInput = new QueryInput();
queryInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE));
queryInput.setFilter(queryFilter);
queryInput.setFilter(filterClone);
queryInput.setSelectDistinct(true);
queryInput.setRecordPipe(getRecordPipe());
queryInput.setLimit(getLimit());
queryInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
new QueryAction().execute(queryInput);
@ -101,8 +126,20 @@ public class ExtractViaQueryStep extends AbstractExtractStep
CountInput countInput = new CountInput();
countInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE));
countInput.setFilter(queryFilter);
countInput.setIncludeDistinctCount(true);
CountOutput countOutput = new CountAction().execute(countInput);
return (countOutput.getCount());
Integer count = countOutput.getDistinctCount();
/////////////////////////////////////////////////////////////////////////////////////////////
// in case the filter we're running has a limit, but the count found more than that limit, //
// well then, just return that limit - as the process won't run on more rows than that. //
/////////////////////////////////////////////////////////////////////////////////////////////
if(count != null & queryFilter.getLimit() != null && count > queryFilter.getLimit())
{
count = queryFilter.getLimit();
}
return count;
}

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java
View File

@ -85,8 +85,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep
queryInput.setFilter(new QQueryFilter()
.withCriteria(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, List.of(script.getValue("id"))))
.withOrderBy(new QFilterOrderBy("sequenceNo", false))
);
queryInput.setLimit(1);
.withLimit(1));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
if(!queryOutput.getRecords().isEmpty())
{

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java
View File

@ -209,8 +209,7 @@ public class GeneralProcessUtils
QueryInput queryInput = new QueryInput();
queryInput.setTableName(tableName);
queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, fieldValue)));
queryInput.setLimit(1);
queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, fieldValue)).withLimit(1));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
return (queryOutput.getRecords().stream().findFirst());
}

21
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java
View File

@ -128,38 +128,31 @@ class EnumerationQueryActionTest extends BaseTest
QueryInput queryInput = new QueryInput();
queryInput.setTableName("statesEnum");
queryInput.setSkip(0);
queryInput.setLimit(null);
queryInput.setFilter(new QQueryFilter().withSkip(0).withLimit(null));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(List.of("Missouri", "Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList());
queryInput.setSkip(1);
queryInput.setLimit(null);
queryInput.setFilter(new QQueryFilter().withSkip(1).withLimit(null));
queryOutput = new QueryAction().execute(queryInput);
assertEquals(List.of("Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList());
queryInput.setSkip(2);
queryInput.setLimit(null);
queryInput.setFilter(new QQueryFilter().withSkip(2).withLimit(null));
queryOutput = new QueryAction().execute(queryInput);
assertEquals(List.of(), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList());
queryInput.setSkip(null);
queryInput.setLimit(1);
queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(1));
queryOutput = new QueryAction().execute(queryInput);
assertEquals(List.of("Missouri"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList());
queryInput.setSkip(null);
queryInput.setLimit(2);
queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(2));
queryOutput = new QueryAction().execute(queryInput);
assertEquals(List.of("Missouri", "Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList());
queryInput.setSkip(null);
queryInput.setLimit(3);
queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(3));
queryOutput = new QueryAction().execute(queryInput);
assertEquals(List.of("Missouri", "Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList());
queryInput.setSkip(null);
queryInput.setLimit(0);
queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(0));
queryOutput = new QueryAction().execute(queryInput);
assertEquals(List.of(), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList());
}

7
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java
View File

@ -341,14 +341,13 @@ class MemoryBackendModuleTest extends BaseTest
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(table.getName());
queryInput.setLimit(2);
queryInput.setFilter(new QQueryFilter().withLimit(2));
assertEquals(2, new QueryAction().execute(queryInput).getRecords().size());
queryInput.setLimit(1);
queryInput.setFilter(new QQueryFilter().withLimit(1));
assertEquals(1, new QueryAction().execute(queryInput).getRecords().size());
queryInput.setSkip(4);
queryInput.setLimit(3);
queryInput.setFilter(new QQueryFilter().withSkip(4).withLimit(3));
assertEquals(0, new QueryAction().execute(queryInput).getRecords().size());
}