From 04a8fa94f9bcbdcdb0f8c461ab9d36cf2c294dfd Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Apr 2023 10:18:44 -0500 Subject: [PATCH] Move skip & limit out of QueryInput, into QQueryFilter... --- .../widgets/ChildRecordListRenderer.java | 2 +- .../scripts/StoreAssociatedScriptAction.java | 3 +- .../SearchPossibleValueSourceAction.java | 2 +- .../actions/tables/query/QQueryFilter.java | 69 ++++++++++++ .../actions/tables/query/QueryInput.java | 102 ++++++------------ .../dashboard/nocode/WidgetQueryField.java | 5 +- .../enumeration/EnumerationQueryAction.java | 2 +- .../implementations/mock/MockQueryAction.java | 5 +- .../utils/BackendQueryFilterUtils.java | 82 +++++++------- .../ExtractViaQueryStep.java | 43 +++++++- .../StoreScriptRevisionProcessStep.java | 3 +- .../processes/utils/GeneralProcessUtils.java | 3 +- .../EnumerationQueryActionTest.java | 21 ++-- .../memory/MemoryBackendModuleTest.java | 7 +- .../module/api/actions/BaseAPIActionUtil.java | 17 +-- .../rdbms/actions/RDBMSQueryAction.java | 10 +- .../qqq/api/javalin/QJavalinApiHandler.java | 7 +- .../javalin/QJavalinImplementation.java | 19 ++-- .../javalin/QJavalinScriptsHandler.java | 3 +- .../picocli/QPicoCliImplementation.java | 7 +- .../qqq/slack/QSlackImplementation.java | 3 +- 21 files changed, 244 insertions(+), 171 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java index 5a068d8d..92cacd2f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java @@ -194,13 +194,13 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(record.getValue(joinOn.getLeftField())))); } filter.setOrderBys(join.getOrderBys()); + filter.setLimit(maxRows); QueryInput queryInput = new QueryInput(); queryInput.setTableName(join.getRightTable()); queryInput.setShouldTranslatePossibleValues(true); queryInput.setShouldGenerateDisplayValues(true); queryInput.setFilter(filter); - queryInput.setLimit(maxRows); QueryOutput queryOutput = new QueryAction().execute(queryInput); QTableMetaData table = input.getInstance().getTable(join.getRightTable()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java index 1d2d8848..c9cc5100 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java @@ -156,8 +156,7 @@ public class StoreAssociatedScriptAction queryInput.setFilter(new QQueryFilter() .withCriteria(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, List.of(script.getValue("id")))) .withOrderBy(new QFilterOrderBy("sequenceNo", false)) - ); - queryInput.setLimit(1); + .withLimit(1)); QueryOutput queryOutput = new QueryAction().execute(queryInput); if(!queryOutput.getRecords().isEmpty()) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java index c034e1f7..8a25136c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java @@ -247,7 +247,7 @@ public class SearchPossibleValueSourceAction queryFilter.setOrderBys(possibleValueSource.getOrderByFields()); // todo - skip & limit as params - queryInput.setLimit(250); + queryFilter.setLimit(250); /////////////////////////////////////////////////////////////////////////////////////////////////////// // if given a default filter, make it the 'top level' filter and the one we just created a subfilter // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java index f89d23e4..6ce122bb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java @@ -47,6 +47,13 @@ public class QQueryFilter implements Serializable, Cloneable private BooleanOperator booleanOperator = BooleanOperator.AND; private List subFilters = new ArrayList<>(); + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // skip & limit are meant to only apply to QueryAction (at least at the initial time they are added here) // + // e.g., they are ignored in CountAction, AggregateAction, etc, where their meanings may be less obvious // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + private Integer skip; + private Integer limit; + /******************************************************************************* @@ -398,4 +405,66 @@ public class QQueryFilter implements Serializable, Cloneable } } + + + /******************************************************************************* + ** Getter for skip + *******************************************************************************/ + public Integer getSkip() + { + return (this.skip); + } + + + + /******************************************************************************* + ** Setter for skip + *******************************************************************************/ + public void setSkip(Integer skip) + { + this.skip = skip; + } + + + + /******************************************************************************* + ** Fluent setter for skip + *******************************************************************************/ + public QQueryFilter withSkip(Integer skip) + { + this.skip = skip; + return (this); + } + + + + /******************************************************************************* + ** Getter for limit + *******************************************************************************/ + public Integer getLimit() + { + return (this.limit); + } + + + + /******************************************************************************* + ** Setter for limit + *******************************************************************************/ + public void setLimit(Integer limit) + { + this.limit = limit; + } + + + + /******************************************************************************* + ** Fluent setter for limit + *******************************************************************************/ + public QQueryFilter withLimit(Integer limit) + { + this.limit = limit; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java index 3a49001e..d1335dbe 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java @@ -39,8 +39,6 @@ public class QueryInput extends AbstractTableActionInput { private QBackendTransaction transaction; private QQueryFilter filter; - private Integer skip; - private Integer limit; private RecordPipe recordPipe; @@ -55,7 +53,8 @@ public class QueryInput extends AbstractTableActionInput ///////////////////////////////////////////////////////////////////////////////////////// private Set fieldsToTranslatePossibleValues; - private List queryJoins = null; + private List queryJoins = null; + private boolean selectDistinct = false; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if you say you want to includeAssociations, you can limit which ones by passing them in associationNamesToInclude. // @@ -98,50 +97,6 @@ public class QueryInput extends AbstractTableActionInput - /******************************************************************************* - ** Getter for skip - ** - *******************************************************************************/ - public Integer getSkip() - { - return skip; - } - - - - /******************************************************************************* - ** Setter for skip - ** - *******************************************************************************/ - public void setSkip(Integer skip) - { - this.skip = skip; - } - - - - /******************************************************************************* - ** Getter for limit - ** - *******************************************************************************/ - public Integer getLimit() - { - return limit; - } - - - - /******************************************************************************* - ** Setter for limit - ** - *******************************************************************************/ - public void setLimit(Integer limit) - { - this.limit = limit; - } - - - /******************************************************************************* ** Getter for recordPipe ** @@ -359,28 +314,6 @@ public class QueryInput extends AbstractTableActionInput - /******************************************************************************* - ** Fluent setter for skip - *******************************************************************************/ - public QueryInput withSkip(Integer skip) - { - this.skip = skip; - return (this); - } - - - - /******************************************************************************* - ** Fluent setter for limit - *******************************************************************************/ - public QueryInput withLimit(Integer limit) - { - this.limit = limit; - return (this); - } - - - /******************************************************************************* ** Fluent setter for recordPipe *******************************************************************************/ @@ -497,4 +430,35 @@ public class QueryInput extends AbstractTableActionInput return (this); } + + + /******************************************************************************* + ** Getter for selectDistinct + *******************************************************************************/ + public boolean getSelectDistinct() + { + return (this.selectDistinct); + } + + + + /******************************************************************************* + ** Setter for selectDistinct + *******************************************************************************/ + public void setSelectDistinct(boolean selectDistinct) + { + this.selectDistinct = selectDistinct; + } + + + + /******************************************************************************* + ** Fluent setter for selectDistinct + *******************************************************************************/ + public QueryInput withSelectDistinct(boolean selectDistinct) + { + this.selectDistinct = selectDistinct; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/WidgetQueryField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/WidgetQueryField.java index f5997b6d..197ce195 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/WidgetQueryField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/WidgetQueryField.java @@ -62,8 +62,9 @@ public class WidgetQueryField extends AbstractWidgetValueSourceWithFilter { QueryInput queryInput = new QueryInput(); queryInput.setTableName(tableName); - queryInput.setFilter(getEffectiveFilter(input)); - queryInput.setLimit(1); + QQueryFilter effectiveFilter = getEffectiveFilter(input); + queryInput.setFilter(effectiveFilter); + effectiveFilter.setLimit(1); QueryOutput queryOutput = new QueryAction().execute(queryInput); if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords())) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryAction.java index 039e0123..c04d1dd1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryAction.java @@ -69,7 +69,7 @@ public class EnumerationQueryAction implements QueryInterface } BackendQueryFilterUtils.sortRecordList(queryInput.getFilter(), recordList); - recordList = BackendQueryFilterUtils.applySkipAndLimit(queryInput, recordList); + recordList = BackendQueryFilterUtils.applySkipAndLimit(queryInput.getFilter(), recordList); QueryOutput queryOutput = new QueryOutput(queryInput); queryOutput.addRecords(recordList); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java index 0d4b918a..d66791f2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java @@ -27,7 +27,6 @@ import java.math.BigDecimal; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.Month; -import java.util.Objects; import java.util.UUID; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -36,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; /******************************************************************************* @@ -59,7 +59,8 @@ public class MockQueryAction implements QueryInterface QueryOutput queryOutput = new QueryOutput(queryInput); - int rows = Objects.requireNonNullElse(queryInput.getLimit(), 1); + @SuppressWarnings("UnnecessaryUnboxing") // force an un-boxing, to force an NPE if it's null, to get to the "else 1" + int rows = ObjectUtils.tryElse(() -> queryInput.getFilter().getLimit().intValue(), 1); for(int i = 0; i < rows; i++) { QRecord record = new QRecord(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java index abf691d8..e05d0bbd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java @@ -33,7 +33,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -129,42 +128,42 @@ public class BackendQueryFilterUtils public static boolean doesCriteriaMatch(QFilterCriteria criterion, String fieldName, Serializable value) { boolean criterionMatches = switch(criterion.getOperator()) + { + case EQUALS -> testEquals(criterion, value); + case NOT_EQUALS -> !testEquals(criterion, value); + case IN -> testIn(criterion, value); + case NOT_IN -> !testIn(criterion, value); + case IS_BLANK -> testBlank(criterion, value); + case IS_NOT_BLANK -> !testBlank(criterion, value); + case CONTAINS -> testContains(criterion, fieldName, value); + case NOT_CONTAINS -> !testContains(criterion, fieldName, value); + case IS_NULL_OR_IN -> testBlank(criterion, value) || testIn(criterion, value); + case LIKE -> testLike(criterion, fieldName, value); + case NOT_LIKE -> !testLike(criterion, fieldName, value); + case STARTS_WITH -> testStartsWith(criterion, fieldName, value); + case NOT_STARTS_WITH -> !testStartsWith(criterion, fieldName, value); + case ENDS_WITH -> testEndsWith(criterion, fieldName, value); + case NOT_ENDS_WITH -> !testEndsWith(criterion, fieldName, value); + case GREATER_THAN -> testGreaterThan(criterion, value); + case GREATER_THAN_OR_EQUALS -> testGreaterThan(criterion, value) || testEquals(criterion, value); + case LESS_THAN -> !testGreaterThan(criterion, value) && !testEquals(criterion, value); + case LESS_THAN_OR_EQUALS -> !testGreaterThan(criterion, value); + case BETWEEN -> { - case EQUALS -> testEquals(criterion, value); - case NOT_EQUALS -> !testEquals(criterion, value); - case IN -> testIn(criterion, value); - case NOT_IN -> !testIn(criterion, value); - case IS_BLANK -> testBlank(criterion, value); - case IS_NOT_BLANK -> !testBlank(criterion, value); - case CONTAINS -> testContains(criterion, fieldName, value); - case NOT_CONTAINS -> !testContains(criterion, fieldName, value); - case IS_NULL_OR_IN -> testBlank(criterion, value) || testIn(criterion, value); - case LIKE -> testLike(criterion, fieldName, value); - case NOT_LIKE -> !testLike(criterion, fieldName, value); - case STARTS_WITH -> testStartsWith(criterion, fieldName, value); - case NOT_STARTS_WITH -> !testStartsWith(criterion, fieldName, value); - case ENDS_WITH -> testEndsWith(criterion, fieldName, value); - case NOT_ENDS_WITH -> !testEndsWith(criterion, fieldName, value); - case GREATER_THAN -> testGreaterThan(criterion, value); - case GREATER_THAN_OR_EQUALS -> testGreaterThan(criterion, value) || testEquals(criterion, value); - case LESS_THAN -> !testGreaterThan(criterion, value) && !testEquals(criterion, value); - case LESS_THAN_OR_EQUALS -> !testGreaterThan(criterion, value); - case BETWEEN -> - { - QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues()); - QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues())); - criteria1.getValues().remove(0); - yield (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value)); - } - case NOT_BETWEEN -> - { - QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues()); - QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues())); - criteria1.getValues().remove(0); - boolean between = (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value)); - yield !between; - } - }; + QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues()); + QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues())); + criteria1.getValues().remove(0); + yield (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value)); + } + case NOT_BETWEEN -> + { + QFilterCriteria criteria0 = new QFilterCriteria().withValues(criterion.getValues()); + QFilterCriteria criteria1 = new QFilterCriteria().withValues(new ArrayList<>(criterion.getValues())); + criteria1.getValues().remove(0); + boolean between = (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value)); + yield !between; + } + }; return criterionMatches; } @@ -524,9 +523,14 @@ public class BackendQueryFilterUtils /******************************************************************************* ** Apply skip & limit attributes from queryInput to a list of records. *******************************************************************************/ - public static List applySkipAndLimit(QueryInput queryInput, List recordList) + public static List applySkipAndLimit(QQueryFilter queryFilter, List recordList) { - Integer skip = queryInput.getSkip(); + if(queryFilter == null) + { + return (recordList); + } + + Integer skip = queryFilter.getSkip(); if(skip != null && skip > 0) { if(skip < recordList.size()) @@ -539,7 +543,7 @@ public class BackendQueryFilterUtils } } - Integer limit = queryInput.getLimit(); + Integer limit = queryFilter.getLimit(); if(limit != null && limit >= 0 && limit < recordList.size()) { recordList = recordList.subList(0, limit); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java index 414b0c54..0b85c841 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java @@ -29,6 +29,7 @@ import java.util.List; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; @@ -52,6 +53,8 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; *******************************************************************************/ public class ExtractViaQueryStep extends AbstractExtractStep { + private static final QLogger LOG = QLogger.getLogger(ExtractViaQueryStep.class); + public static final String FIELD_SOURCE_TABLE = "sourceTable"; private QQueryFilter queryFilter; @@ -77,11 +80,33 @@ public class ExtractViaQueryStep extends AbstractExtractStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { + ////////////////////////////////////////////////////////////////// + // clone the filter, since we're going to edit it (set a limit) // + ////////////////////////////////////////////////////////////////// + QQueryFilter filterClone = queryFilter.clone(); + + ////////////////////////////////////////////////////////////////////////////////////////////// + // if there's a limit in the extract step (e.g., the 20-record limit on the preview screen) // + // then set that limit in the filter - UNLESS - there's already a limit in the filter for // + // a smaller number of records. // + ////////////////////////////////////////////////////////////////////////////////////////////// + if(getLimit() != null) + { + if(filterClone.getLimit() != null && filterClone.getLimit() < getLimit()) + { + LOG.trace("Using filter's limit [" + filterClone.getLimit() + "] rather than step's limit [" + getLimit() + "]"); + } + else + { + filterClone.setLimit(getLimit()); + } + } + QueryInput queryInput = new QueryInput(); queryInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE)); - queryInput.setFilter(queryFilter); + queryInput.setFilter(filterClone); + queryInput.setSelectDistinct(true); queryInput.setRecordPipe(getRecordPipe()); - queryInput.setLimit(getLimit()); queryInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback()); new QueryAction().execute(queryInput); @@ -101,8 +126,20 @@ public class ExtractViaQueryStep extends AbstractExtractStep CountInput countInput = new CountInput(); countInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE)); countInput.setFilter(queryFilter); + countInput.setIncludeDistinctCount(true); CountOutput countOutput = new CountAction().execute(countInput); - return (countOutput.getCount()); + Integer count = countOutput.getDistinctCount(); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // in case the filter we're running has a limit, but the count found more than that limit, // + // well then, just return that limit - as the process won't run on more rows than that. // + ///////////////////////////////////////////////////////////////////////////////////////////// + if(count != null & queryFilter.getLimit() != null && count > queryFilter.getLimit()) + { + count = queryFilter.getLimit(); + } + + return count; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java index d1a0fb82..b887ff9a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java @@ -85,8 +85,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep queryInput.setFilter(new QQueryFilter() .withCriteria(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, List.of(script.getValue("id")))) .withOrderBy(new QFilterOrderBy("sequenceNo", false)) - ); - queryInput.setLimit(1); + .withLimit(1)); QueryOutput queryOutput = new QueryAction().execute(queryInput); if(!queryOutput.getRecords().isEmpty()) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java index 4cde210b..98a41a4f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java @@ -209,8 +209,7 @@ public class GeneralProcessUtils QueryInput queryInput = new QueryInput(); queryInput.setTableName(tableName); - queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, fieldValue))); - queryInput.setLimit(1); + queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, fieldValue)).withLimit(1)); QueryOutput queryOutput = new QueryAction().execute(queryInput); return (queryOutput.getRecords().stream().findFirst()); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java index d1b13a98..909c9f7c 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java @@ -128,38 +128,31 @@ class EnumerationQueryActionTest extends BaseTest QueryInput queryInput = new QueryInput(); queryInput.setTableName("statesEnum"); - queryInput.setSkip(0); - queryInput.setLimit(null); + queryInput.setFilter(new QQueryFilter().withSkip(0).withLimit(null)); QueryOutput queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of("Missouri", "Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(1); - queryInput.setLimit(null); + queryInput.setFilter(new QQueryFilter().withSkip(1).withLimit(null)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of("Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(2); - queryInput.setLimit(null); + queryInput.setFilter(new QQueryFilter().withSkip(2).withLimit(null)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of(), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(null); - queryInput.setLimit(1); + queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(1)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of("Missouri"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(null); - queryInput.setLimit(2); + queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(2)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of("Missouri", "Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(null); - queryInput.setLimit(3); + queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(3)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of("Missouri", "Illinois"), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); - queryInput.setSkip(null); - queryInput.setLimit(0); + queryInput.setFilter(new QQueryFilter().withSkip(null).withLimit(0)); queryOutput = new QueryAction().execute(queryInput); assertEquals(List.of(), queryOutput.getRecords().stream().map(r -> r.getValueString("name")).toList()); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java index 34a25a95..23b8ab93 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java @@ -341,14 +341,13 @@ class MemoryBackendModuleTest extends BaseTest { QueryInput queryInput = new QueryInput(); queryInput.setTableName(table.getName()); - queryInput.setLimit(2); + queryInput.setFilter(new QQueryFilter().withLimit(2)); assertEquals(2, new QueryAction().execute(queryInput).getRecords().size()); - queryInput.setLimit(1); + queryInput.setFilter(new QQueryFilter().withLimit(1)); assertEquals(1, new QueryAction().execute(queryInput).getRecords().size()); - queryInput.setSkip(4); - queryInput.setLimit(3); + queryInput.setFilter(new QQueryFilter().withSkip(4).withLimit(3)); assertEquals(0, new QueryAction().execute(queryInput).getRecords().size()); } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index d8ecf17a..3f0a230c 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -228,10 +228,12 @@ public class BaseAPIActionUtil *******************************************************************************/ public QueryOutput doQuery(QTableMetaData table, QueryInput queryInput) throws QException { - QueryOutput queryOutput = new QueryOutput(queryInput); - Integer originalLimit = queryInput.getLimit(); - Integer limit = originalLimit; - Integer skip = queryInput.getSkip(); + QueryOutput queryOutput = new QueryOutput(queryInput); + QQueryFilter filter = queryInput.getFilter(); + + Integer originalLimit = filter == null ? null : filter.getLimit(); + Integer limit = originalLimit; + Integer skip = filter == null ? null : filter.getSkip(); if(limit == null) { @@ -243,10 +245,9 @@ public class BaseAPIActionUtil { try { - QQueryFilter filter = queryInput.getFilter(); - String paramString = buildQueryStringForGet(filter, limit, skip, table.getFields()); - String url = buildTableUrl(table) + paramString; - HttpGet request = new HttpGet(url); + String paramString = buildQueryStringForGet(filter, limit, skip, table.getFields()); + String url = buildTableUrl(table) + paramString; + HttpGet request = new HttpGet(url); QHttpResponse response = makeRequest(table, request); int count = processGetResponse(table, response, queryOutput); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 412dd35a..4522282f 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -83,14 +83,14 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf sql.append(" ORDER BY ").append(makeOrderByClause(table, filter.getOrderBys(), joinsContext)); } - if(queryInput.getLimit() != null) + if(filter != null && filter.getLimit() != null) { - sql.append(" LIMIT ").append(queryInput.getLimit()); + sql.append(" LIMIT ").append(filter.getLimit()); - if(queryInput.getSkip() != null) + if(filter.getSkip() != null) { // todo - other sql grammars? - sql.append(" OFFSET ").append(queryInput.getSkip()); + sql.append(" OFFSET ").append(filter.getSkip()); } } @@ -200,7 +200,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf List queryJoins = queryInput.getQueryJoins(); QTableMetaData table = instance.getTable(tableName); - boolean requiresDistinct = doesSelectClauseRequireDistinct(table); + boolean requiresDistinct = queryInput.getSelectDistinct() || doesSelectClauseRequireDistinct(table); String clausePrefix = (requiresDistinct) ? "SELECT DISTINCT " : "SELECT "; List fieldList = new ArrayList<>(table.getFields().values()); diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java index 0cb92760..72222593 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java @@ -913,6 +913,8 @@ public class QJavalinApiHandler PermissionsHelper.checkTablePermissionThrowing(queryInput, TablePermissionSubType.READ); + filter = new QQueryFilter(); + Integer pageSize = 50; if(StringUtils.hasContent(context.queryParam("pageSize"))) { @@ -947,12 +949,11 @@ public class QJavalinApiHandler badRequestMessages.add("pageNo must be greater than 0."); } - queryInput.setLimit(pageSize); - queryInput.setSkip((pageNo - 1) * pageSize); + filter.setLimit(pageSize); + filter.setSkip((pageNo - 1) * pageSize); // queryInput.setQueryJoins(processQueryJoinsParam(context)); - filter = new QQueryFilter(); if("and".equalsIgnoreCase(context.queryParam("booleanOperator"))) { filter.setBooleanOperator(QQueryFilter.BooleanOperator.AND); diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index a8eaae97..ddf3324c 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -843,8 +843,6 @@ public class QJavalinImplementation queryInput.setTableName(table); queryInput.setShouldGenerateDisplayValues(true); queryInput.setShouldTranslatePossibleValues(true); - queryInput.setSkip(QJavalinUtils.integerQueryParam(context, "skip")); - queryInput.setLimit(QJavalinUtils.integerQueryParam(context, "limit")); PermissionsHelper.checkTablePermissionThrowing(queryInput, TablePermissionSubType.READ); @@ -858,15 +856,22 @@ public class QJavalinImplementation queryInput.setFilter(JsonUtils.toObject(filter, QQueryFilter.class)); } + Integer skip = QJavalinUtils.integerQueryParam(context, "skip"); + Integer limit = QJavalinUtils.integerQueryParam(context, "limit"); + if(skip != null || limit != null) + { + if(queryInput.getFilter() == null) + { + queryInput.setFilter(new QQueryFilter()); + } + queryInput.getFilter().setSkip(skip); + queryInput.getFilter().setLimit(limit); + } + queryInput.setQueryJoins(processQueryJoinsParam(context)); QueryAction queryAction = new QueryAction(); QueryOutput queryOutput = queryAction.execute(queryInput); - int rowIndex = 0; - for(QRecord record : queryOutput.getRecords()) - { - record.setValue("__qRowIndex", rowIndex++); - } QJavalinAccessLogger.logEndSuccess(logPair("recordCount", queryOutput.getRecords().size()), logPairIfSlow("filter", filter, SLOW_LOG_THRESHOLD_MS)); context.result(JsonUtils.toJson(queryOutput)); diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinScriptsHandler.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinScriptsHandler.java index 61f84bfa..b8cfb7b2 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinScriptsHandler.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinScriptsHandler.java @@ -254,8 +254,7 @@ public class QJavalinScriptsHandler queryInput.setFilter(new QQueryFilter() .withCriteria(new QFilterCriteria("scriptRevisionId", QCriteriaOperator.EQUALS, List.of(scriptRevisionId))) .withOrderBy(new QFilterOrderBy("id", false)) - ); - queryInput.setLimit(100); + .withLimit(100)); QueryOutput queryOutput = new QueryAction().execute(queryInput); if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords())) diff --git a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java index b636d735..093a5f0d 100644 --- a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java +++ b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java @@ -531,7 +531,6 @@ public class QPicoCliImplementation { QueryInput queryInput = new QueryInput(); queryInput.setTableName(tableName); - queryInput.setSkip(subParseResult.matchedOptionValue("skip", null)); // todo - think about these (e.g., based on user's requested output format? // queryInput.setShouldGenerateDisplayValues(true); @@ -553,6 +552,8 @@ public class QPicoCliImplementation .withValues(List.of(primaryKeyValue))); queryInput.setFilter(filter); + filter.setSkip(subParseResult.matchedOptionValue("skip", null)); + QueryAction queryAction = new QueryAction(); QueryOutput queryOutput = queryAction.execute(queryInput); List records = queryOutput.getRecords(); @@ -577,9 +578,9 @@ public class QPicoCliImplementation { QueryInput queryInput = new QueryInput(); queryInput.setTableName(tableName); - queryInput.setSkip(subParseResult.matchedOptionValue("skip", null)); - queryInput.setLimit(subParseResult.matchedOptionValue("limit", null)); queryInput.setFilter(generateQueryFilter(subParseResult)); + queryInput.getFilter().setSkip(subParseResult.matchedOptionValue("skip", null)); + queryInput.getFilter().setLimit(subParseResult.matchedOptionValue("limit", null)); // todo - think about these (e.g., based on user's requested output format? // queryInput.setShouldGenerateDisplayValues(true); diff --git a/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java b/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java index 82527128..56ee591c 100644 --- a/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java +++ b/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java @@ -52,6 +52,7 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; @@ -384,7 +385,7 @@ public class QSlackImplementation try { QueryInput queryInput = new QueryInput(); - queryInput.setLimit(10); + queryInput.setFilter(new QQueryFilter().withLimit(10)); queryInput.setTableName(tableName); setupSession(context, queryInput); QueryOutput output = new QueryAction().execute(queryInput);