From 82810c2b661002e16f03560aa31347caa6c46117 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Oct 2022 12:24:35 -0500 Subject: [PATCH] Add ExtractViaBasepullQueryStep; add pagination & piping to api query --- .../actions/processes/RunProcessAction.java | 30 ++- .../core/instances/QInstanceEnricher.java | 18 ++ .../processes/RunBackendStepInput.java | 11 + .../actions/tables/query/QFilterCriteria.java | 47 ++++ .../actions/tables/query/QFilterOrderBy.java | 10 + .../actions/tables/query/QQueryFilter.java | 41 +++ .../metadata/processes/QProcessMetaData.java | 2 +- .../model/metadata/tables/QTableMetaData.java | 1 + .../basepull/BasepullConfiguration.java | 247 ++++++++++++++++++ .../basepull/ExtractViaBasepullQueryStep.java | 82 ++++++ .../instances/QInstanceValidatorTest.java | 7 +- .../ExtractViaBasepullQueryStepTest.java | 74 ++++++ .../qqq/backend/core/utils/TestUtils.java | 2 +- .../module/api/actions/APIQueryAction.java | 100 +++++-- .../module/api/actions/BaseAPIActionUtil.java | 41 ++- qqq-dev-tools/bin/reset-snapshot-deps.sh | 6 +- qqq-dev-tools/bin/update-all-qqq-deps.sh | 10 +- qqq-dev-tools/bin/update-dep.sh | 14 +- 18 files changed, 689 insertions(+), 54 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/BasepullConfiguration.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStepTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index 035ec2ad..e5ef9755 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -52,7 +52,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.general.BasepullConfiguration; +import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider; import com.kingsrook.qqq.backend.core.state.StateProviderInterface; import com.kingsrook.qqq.backend.core.state.StateType; @@ -69,9 +69,11 @@ import org.apache.logging.log4j.Logger; *******************************************************************************/ public class RunProcessAction { - private static final Logger LOG = LogManager.getLogger(RunProcessAction.class); - public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey"; - public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey"; + private static final Logger LOG = LogManager.getLogger(RunProcessAction.class); + + public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey"; + public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey"; + public static final String BASEPULL_TIMESTAMP_FIELD = "basepullTimestampField"; @@ -422,7 +424,7 @@ public class RunProcessAction /******************************************************************************* - ** + ** Insert or update the last runtime value for this basepull into the backend. *******************************************************************************/ protected void storeLastRunTime(RunProcessInput runProcessInput, QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException { @@ -489,13 +491,22 @@ public class RunProcessAction /******************************************************************************* - ** + ** Lookup the last runtime for this basepull, and set it (plus now) in the process's + ** values. *******************************************************************************/ protected void persistLastRunTime(RunProcessInput runProcessInput, QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException { - //////////////////////////////////////////////////////////////////////////////////////////////// - // store 'now', which will be used to update basepull record if process completes sucessfully // - //////////////////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////// + // if these values were already computed, don't re-do // + //////////////////////////////////////////////////////// + if(runProcessInput.getValue(BASEPULL_THIS_RUNTIME_KEY) != null) + { + return; + } + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // store 'now', which will be used to update basepull record if process completes successfully // + ///////////////////////////////////////////////////////////////////////////////////////////////// Instant now = Instant.now(); runProcessInput.getValues().put(BASEPULL_THIS_RUNTIME_KEY, now); @@ -533,5 +544,6 @@ public class RunProcessAction } runProcessInput.getValues().put(BASEPULL_LAST_RUNTIME_KEY, lastRunTime); + runProcessInput.getValues().put(BASEPULL_TIMESTAMP_FIELD, basepullConfiguration.getTimestampField()); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 573b8ba2..6f11bd5b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -316,6 +316,24 @@ public class QInstanceEnricher { generateAppSections(app); } + + for(QAppSection section : CollectionUtils.nonNullList(app.getSections())) + { + enrichAppSection(section); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void enrichAppSection(QAppSection section) + { + if(!StringUtils.hasContent(section.getLabel())) + { + section.setLabel(nameToLabel(section.getName())); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index 3c93796a..0ffbd612 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -419,6 +419,17 @@ public class RunBackendStepInput extends AbstractActionInput + /******************************************************************************* + ** Getter for a single field's value + ** + *******************************************************************************/ + public Instant getValueInstant(String fieldName) + { + return (ValueUtils.getValueAsInstant(getValue(fieldName))); + } + + + /******************************************************************************* ** Accessor for processState - protected, because we generally want to access ** its members through wrapper methods, we think diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java index 51653a6b..f83e959d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java @@ -25,6 +25,9 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -33,6 +36,8 @@ import java.util.List; *******************************************************************************/ public class QFilterCriteria implements Serializable, Cloneable { + private static final Logger LOG = LogManager.getLogger(QFilterCriteria.class); + private String fieldName; private QCriteriaOperator operator; private List values; @@ -183,4 +188,46 @@ public class QFilterCriteria implements Serializable, Cloneable this.values = values; return this; } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + StringBuilder rs = new StringBuilder(fieldName); + try + { + rs.append(" ").append(operator).append(" "); + if(CollectionUtils.nullSafeHasContents(values)) + { + if(values.size() == 1) + { + rs.append(values.get(0)); + } + else + { + int index = 0; + for(Serializable value : values) + { + if(index++ > 9) + { + rs.append("and ").append(values.size() - index).append(" more"); + break; + } + rs.append(value).append(","); + } + } + } + } + catch(Exception e) + { + LOG.warn("Error in toString", e); + rs.append("Error generating toString..."); + } + + return (rs.toString()); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterOrderBy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterOrderBy.java index 4cfa6395..52eca98c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterOrderBy.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterOrderBy.java @@ -151,4 +151,14 @@ public class QFilterOrderBy implements Serializable, Cloneable return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return (fieldName + " " + (isAscending ? "ASC" : "DESC")); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java index f39403de..6d1fc376 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java @@ -26,6 +26,8 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -34,6 +36,8 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; *******************************************************************************/ public class QQueryFilter implements Serializable, Cloneable { + private static final Logger LOG = LogManager.getLogger(QQueryFilter.class); + private List criteria = new ArrayList<>(); private List orderBys = new ArrayList<>(); @@ -301,4 +305,41 @@ public class QQueryFilter implements Serializable, Cloneable subFilters.add(subFilter); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + StringBuilder rs = new StringBuilder("("); + try + { + for(QFilterCriteria criterion : CollectionUtils.nonNullList(criteria)) + { + rs.append(criterion).append(" ").append(getBooleanOperator()); + } + + for(QQueryFilter subFilter : CollectionUtils.nonNullList(subFilters)) + { + rs.append(subFilter); + } + rs.append(")"); + + rs.append("OrderBy["); + for(QFilterOrderBy orderBy : orderBys) + { + rs.append(orderBy).append(","); + } + rs.append("]"); + } + catch(Exception e) + { + LOG.warn("Error in toString", e); + rs.append("Error generating toString..."); + } + + return (rs.toString()); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index 7ca1dae3..cc54e526 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -33,7 +33,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.general.BasepullConfiguration; +import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; /******************************************************************************* diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index 48ca0556..7c1f0571 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -822,4 +822,5 @@ public class QTableMetaData implements QAppChildMetaData, Serializable return (this); } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/BasepullConfiguration.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/BasepullConfiguration.java new file mode 100644 index 00000000..e207258d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/BasepullConfiguration.java @@ -0,0 +1,247 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.basepull; + + +import java.io.Serializable; + + +/******************************************************************************* + ** Class for storing all basepull configuration data + ** + *******************************************************************************/ +public class BasepullConfiguration implements Serializable +{ + private String tableName; // the table that stores the basepull timestamps + private String keyField; // the field in the basepull timestamps table that stores the key of the basepull (e.g., a process name) + private String keyValue; // the key applied to the keyField - optional - if not set, process.getName is used. + + private String lastRunTimeFieldName; // the field in the basepull timestamps table that stores the last-run time for the job. + private Integer hoursBackForInitialTimestamp; // for the first-run use-case (where there is no row in the timestamps table), how many hours back in time to look. + + private String timestampField; // the name of the field in the table being queried against the last-run timestamp. + + + + /******************************************************************************* + ** Getter for tableName + ** + *******************************************************************************/ + public String getTableName() + { + return tableName; + } + + + + /******************************************************************************* + ** Setter for tableName + ** + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + ** + *******************************************************************************/ + public BasepullConfiguration withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for keyField + ** + *******************************************************************************/ + public String getKeyField() + { + return keyField; + } + + + + /******************************************************************************* + ** Setter for keyField + ** + *******************************************************************************/ + public void setKeyField(String keyField) + { + this.keyField = keyField; + } + + + + /******************************************************************************* + ** Fluent setter for keyField + ** + *******************************************************************************/ + public BasepullConfiguration withKeyField(String keyField) + { + this.keyField = keyField; + return (this); + } + + + + /******************************************************************************* + ** Getter for keyValue + ** + *******************************************************************************/ + public String getKeyValue() + { + return keyValue; + } + + + + /******************************************************************************* + ** Setter for keyValue + ** + *******************************************************************************/ + public void setKeyValue(String keyValue) + { + this.keyValue = keyValue; + } + + + + /******************************************************************************* + ** Fluent setter for keyValue + ** + *******************************************************************************/ + public BasepullConfiguration withKeyValue(String keyValue) + { + this.keyValue = keyValue; + return (this); + } + + + + /******************************************************************************* + ** Getter for lastRunTimeFieldName + ** + *******************************************************************************/ + public String getLastRunTimeFieldName() + { + return lastRunTimeFieldName; + } + + + + /******************************************************************************* + ** Setter for lastRunTimeFieldName + ** + *******************************************************************************/ + public void setLastRunTimeFieldName(String lastRunTimeFieldName) + { + this.lastRunTimeFieldName = lastRunTimeFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for lastRunTimeFieldName + ** + *******************************************************************************/ + public BasepullConfiguration withLastRunTimeFieldName(String lastRunTimeFieldName) + { + this.lastRunTimeFieldName = lastRunTimeFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for hoursBackForInitialTimestamp + ** + *******************************************************************************/ + public Integer getHoursBackForInitialTimestamp() + { + return hoursBackForInitialTimestamp; + } + + + + /******************************************************************************* + ** Setter for hoursBackForInitialTimestamp + ** + *******************************************************************************/ + public void setHoursBackForInitialTimestamp(Integer hoursBackForInitialTimestamp) + { + this.hoursBackForInitialTimestamp = hoursBackForInitialTimestamp; + } + + + + /******************************************************************************* + ** Fluent setter for hoursBackForInitialTimestamp + ** + *******************************************************************************/ + public BasepullConfiguration withHoursBackForInitialTimestamp(Integer hoursBackForInitialTimestamp) + { + this.hoursBackForInitialTimestamp = hoursBackForInitialTimestamp; + return (this); + } + + + + /******************************************************************************* + ** Getter for timestampField + ** + *******************************************************************************/ + public String getTimestampField() + { + return timestampField; + } + + + + /******************************************************************************* + ** Setter for timestampField + ** + *******************************************************************************/ + public void setTimestampField(String timestampField) + { + this.timestampField = timestampField; + } + + + + /******************************************************************************* + ** Fluent setter for timestampField + ** + *******************************************************************************/ + public BasepullConfiguration withTimestampField(String timestampField) + { + this.timestampField = timestampField; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java new file mode 100644 index 00000000..9e2654be --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java @@ -0,0 +1,82 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.basepull; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; + + +/******************************************************************************* + ** Version of ExtractViaQueryStep that knows how to set up a basepull query. + *******************************************************************************/ +public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep +{ + + /******************************************************************************* + ** + *******************************************************************************/ + protected QQueryFilter getQueryFilter(RunBackendStepInput runBackendStepInput) throws QException + { + ////////////////////////////////////////////////////////////// + // get input query filter or if not found, create a new one // + ////////////////////////////////////////////////////////////// + QQueryFilter queryFilter = new QQueryFilter(); + try + { + queryFilter = super.getQueryFilter(runBackendStepInput); + } + catch(QException qe) + { + /////////////////////////////////////////////////////////////////////////////////////// + // if we catch here, assume that is because there was no default filter, continue on // + /////////////////////////////////////////////////////////////////////////////////////// + } + + /////////////////////////////////////////////////////////////////////////////////////////////////// + // build up a query filter that is against the source table for the given source table timestamp // + // field, finding any records that need processed. // + // query will be for: timestamp > lastRun AND timestamp <= thisRun. // + // then thisRun will be stored, so the next run shouldn't find any records from thisRun. // + /////////////////////////////////////////////////////////////////////////////////////////////////// + queryFilter.addCriteria(new QFilterCriteria() + .withFieldName(runBackendStepInput.getValueString(RunProcessAction.BASEPULL_TIMESTAMP_FIELD)) + .withOperator(QCriteriaOperator.GREATER_THAN) + .withValues(List.of(runBackendStepInput.getBasepullLastRunTime()))); + + queryFilter.addCriteria(new QFilterCriteria() + .withFieldName(runBackendStepInput.getValueString(RunProcessAction.BASEPULL_TIMESTAMP_FIELD)) + .withOperator(QCriteriaOperator.LESS_THAN_OR_EQUALS) + .withValues(List.of(runBackendStepInput.getValueInstant(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY)))); + + queryFilter.addOrderBy(new QFilterOrderBy(runBackendStepInput.getValueString(RunProcessAction.BASEPULL_TIMESTAMP_FIELD))); + + return (queryFilter); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index d6da4b05..ec8246ca 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -680,10 +680,13 @@ class QInstanceValidatorTest @Test void testAppSectionsMissingLabel() { + /////////////////////////////////////////////////////////////////////////////////// + // the enricher makes a label from the name, so, we'll just make them both null. // + /////////////////////////////////////////////////////////////////////////////////// QAppMetaData app = new QAppMetaData().withName("test") .withChild(new QTableMetaData().withName("test")) - .withSection(new QAppSection("Section 1", null, new QIcon("person"), List.of("test"), null, null)); - assertValidationFailureReasons((qInstance) -> qInstance.addApp(app), "Missing a label"); + .withSection(new QAppSection(null, null, new QIcon("person"), List.of("test"), null, null)); + assertValidationFailureReasons((qInstance) -> qInstance.addApp(app), "Missing a label", "Missing a name"); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStepTest.java new file mode 100644 index 00000000..9f4a2827 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStepTest.java @@ -0,0 +1,74 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.basepull; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for ExtractViaBasepullQueryStep + *******************************************************************************/ +class ExtractViaBasepullQueryStepTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + Instant timestamp = Instant.parse("1980-05-31T15:36:00Z"); + Instant now = Instant.now(); + + RunBackendStepInput input = new RunBackendStepInput(TestUtils.defineInstance()); + input.addValue(RunProcessAction.BASEPULL_TIMESTAMP_FIELD, "createDate"); + input.addValue(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY, now); + input.setBasepullLastRunTime(timestamp); + QQueryFilter queryFilter = new ExtractViaBasepullQueryStep().getQueryFilter(input); + + System.out.println(queryFilter); + + assertEquals(2, queryFilter.getCriteria().size()); + assertEquals("createDate", queryFilter.getCriteria().get(0).getFieldName()); + assertEquals(QCriteriaOperator.GREATER_THAN, queryFilter.getCriteria().get(0).getOperator()); + assertEquals(timestamp, queryFilter.getCriteria().get(0).getValues().get(0)); + + assertEquals("createDate", queryFilter.getCriteria().get(1).getFieldName()); + assertEquals(QCriteriaOperator.LESS_THAN_OR_EQUALS, queryFilter.getCriteria().get(1).getOperator()); + assertEquals(now, queryFilter.getCriteria().get(1).getValues().get(0)); + + assertEquals(1, queryFilter.getOrderBys().size()); + assertEquals("createDate", queryFilter.getOrderBys().get(0).getFieldName()); + assertTrue(queryFilter.getOrderBys().get(0).getIsAscending()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index e3cad2eb..defd1006 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -87,9 +87,9 @@ import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationM import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule; +import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; -import com.kingsrook.qqq.backend.core.processes.implementations.general.BasepullConfiguration; import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIQueryAction.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIQueryAction.java index 29afefc6..2f5cc8d8 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIQueryAction.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIQueryAction.java @@ -22,13 +22,11 @@ package com.kingsrook.qqq.backend.module.api.actions; -import java.util.List; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; @@ -55,33 +53,83 @@ public class APIQueryAction extends AbstractAPIAction implements QueryInterface QTableMetaData table = queryInput.getTable(); preAction(queryInput); - try + QueryOutput queryOutput = new QueryOutput(queryInput); + Integer originalLimit = queryInput.getLimit(); + Integer limit = originalLimit; + Integer skip = queryInput.getSkip(); + + if(limit == null) { - QQueryFilter filter = queryInput.getFilter(); - String paramString = apiActionUtil.buildQueryStringForGet(filter, queryInput.getLimit(), queryInput.getSkip(), table.getFields()); - - HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); - HttpClient client = httpClientBuilder.build(); - - String url = apiActionUtil.buildTableUrl(table) + paramString; - LOG.info("API URL: " + url); - HttpGet request = new HttpGet(url); - - apiActionUtil.setupAuthorizationInRequest(request); - apiActionUtil.setupContentTypeInRequest(request); - apiActionUtil.setupAdditionalHeaders(request); - - HttpResponse response = client.execute(request); - List queryResults = apiActionUtil.processGetResponse(table, response); - - QueryOutput queryOutput = new QueryOutput(queryInput); - queryOutput.addRecords(queryResults); - return (queryOutput); + limit = apiActionUtil.getApiStandardLimit(); } - catch(Exception e) + + int totalCount = 0; + while(true) { - LOG.warn("Error in API Query", e); - throw new QException("Error executing query: " + e.getMessage(), e); + try + { + QQueryFilter filter = queryInput.getFilter(); + String paramString = apiActionUtil.buildQueryStringForGet(filter, limit, skip, table.getFields()); + + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); + HttpClient client = httpClientBuilder.build(); + + String url = apiActionUtil.buildTableUrl(table) + paramString; + LOG.info("API URL: " + url); + + /////////////////////////// + // todo - 429 handling!! // + /////////////////////////// + HttpGet request = new HttpGet(url); + + apiActionUtil.setupAuthorizationInRequest(request); + apiActionUtil.setupContentTypeInRequest(request); + apiActionUtil.setupAdditionalHeaders(request); + + HttpResponse response = client.execute(request); + + int count = apiActionUtil.processGetResponse(table, response, queryOutput); + totalCount += count; + + ///////////////////////////////////////////////////////////////////////// + // if we've fetched at least as many as the original limit, then break // + ///////////////////////////////////////////////////////////////////////// + if(originalLimit != null && totalCount >= originalLimit) + { + return (queryOutput); + } + + //////////////////////////////////////////////////////////////////////////////////// + // if we got back less than a full page this time, then we must be done, so break // + //////////////////////////////////////////////////////////////////////////////////// + if(count == 0 || (limit != null && count < limit)) + { + return (queryOutput); + } + + /////////////////////////////////////////////////////////////////// + // if there's an async callback that says we're cancelled, break // + /////////////////////////////////////////////////////////////////// + if(queryInput.getAsyncJobCallback().wasCancelRequested()) + { + LOG.info("Breaking query job, as requested."); + return (queryOutput); + } + + //////////////////////////////////////////////////////////////////////////// + // else, increment the skip by the count we just got, and query for more. // + //////////////////////////////////////////////////////////////////////////// + if(skip == null) + { + skip = 0; + } + skip += count; + } + catch(Exception e) + { + LOG.warn("Error in API Query", e); + throw new QException("Error executing query: " + e.getMessage(), e); + } } } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index a6abb0af..b9f4f935 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.io.Serializable; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Base64; import java.util.List; import java.util.Map; @@ -35,6 +34,8 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -74,7 +75,7 @@ public class BaseAPIActionUtil *******************************************************************************/ public long getMillisToSleepAfterEveryCall() { - return 0; + return (0); } @@ -84,7 +85,7 @@ public class BaseAPIActionUtil *******************************************************************************/ public int getInitialRateLimitBackoffMillis() { - return 0; + return (0); } @@ -94,7 +95,17 @@ public class BaseAPIActionUtil *******************************************************************************/ public int getMaxAllowedRateLimitErrors() { - return 0; + return (0); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Integer getApiStandardLimit() + { + return (20); } @@ -272,7 +283,7 @@ public class BaseAPIActionUtil /******************************************************************************* ** *******************************************************************************/ - protected List processGetResponse(QTableMetaData table, HttpResponse response) throws IOException + protected int processGetResponse(QTableMetaData table, HttpResponse response, QueryOutput queryOutput) throws IOException { int statusCode = response.getStatusLine().getStatusCode(); System.out.println(statusCode); @@ -285,7 +296,7 @@ public class BaseAPIActionUtil HttpEntity entity = response.getEntity(); String resultString = EntityUtils.toString(entity); - List recordList = new ArrayList<>(); + int count = 0; if(StringUtils.hasContent(resultString) && !resultString.equals("null")) { JSONArray resultList = null; @@ -309,16 +320,18 @@ public class BaseAPIActionUtil { for(int i = 0; i < resultList.length(); i++) { - recordList.add(jsonObjectToRecord(resultList.getJSONObject(i), table.getFields())); + queryOutput.addRecord(jsonObjectToRecord(resultList.getJSONObject(i), table.getFields())); + count++; } } else { - recordList.add(jsonObjectToRecord(jsonObject, table.getFields())); + queryOutput.addRecord(jsonObjectToRecord(jsonObject, table.getFields())); + count++; } } - return (recordList); + return (count); } @@ -483,8 +496,14 @@ public class BaseAPIActionUtil *******************************************************************************/ public Integer processGetResponseForCount(QTableMetaData table, HttpResponse response) throws IOException { - List queryResults = processGetResponse(table, response); - return (queryResults.size()); + ///////////////////////////////////////////////////////////////////////////////////////// + // set up a query output with a blank query input - e.g., one that isn't using a pipe. // + ///////////////////////////////////////////////////////////////////////////////////////// + QueryOutput queryOutput = new QueryOutput(new QueryInput()); + processGetResponse(table, response, queryOutput); + List records = queryOutput.getRecords(); + + return (records == null ? null : records.size()); } } diff --git a/qqq-dev-tools/bin/reset-snapshot-deps.sh b/qqq-dev-tools/bin/reset-snapshot-deps.sh index 77240dee..ddfe2aa1 100755 --- a/qqq-dev-tools/bin/reset-snapshot-deps.sh +++ b/qqq-dev-tools/bin/reset-snapshot-deps.sh @@ -9,5 +9,9 @@ CURRENT_VERSION="$(cat $QQQ_DEV_TOOLS_DIR/CURRENT-SNAPSHOT-VERSION)" MODULE_LIST_FILE=$QQQ_DEV_TOOLS_DIR/MODULE_LIST for artifact in $(cat $MODULE_LIST_FILE); do - update-dep.sh $artifact ${CURRENT_VERSION}-SNAPSHOT + update-dep.sh $artifact ${CURRENT_VERSION}-SNAPSHOT -q done + +echo +echo git diff pom.xml +git diff pom.xml diff --git a/qqq-dev-tools/bin/update-all-qqq-deps.sh b/qqq-dev-tools/bin/update-all-qqq-deps.sh index 09cad332..4e19bcd4 100755 --- a/qqq-dev-tools/bin/update-all-qqq-deps.sh +++ b/qqq-dev-tools/bin/update-all-qqq-deps.sh @@ -9,6 +9,14 @@ CURRENT_VERSION="$(cat $QQQ_DEV_TOOLS_DIR/CURRENT-SNAPSHOT-VERSION)" MODULE_LIST_FILE=$QQQ_DEV_TOOLS_DIR/MODULE_LIST for module in $(cat $MODULE_LIST_FILE); do + echo "Updating $module..." version=$(get-latest-snapshot.sh $module $CURRENT_VERSION) - update-dep.sh $module $version + update-dep.sh $module $version -q done + +echo +echo git diff pom.xml +git diff pom.xml +newVersion=$(grep -A1 qqq-backend-core pom.xml | tail -1 | sed 's/.*//;s/<\/version>.*//;s/-........\......./ snapshot/') +echo "You might want to commit that with:" +echo " git commit -m \"Updated qqq deps to $newVersion\" pom.xml" diff --git a/qqq-dev-tools/bin/update-dep.sh b/qqq-dev-tools/bin/update-dep.sh index 22b6ae7d..1fdf54d2 100755 --- a/qqq-dev-tools/bin/update-dep.sh +++ b/qqq-dev-tools/bin/update-dep.sh @@ -8,6 +8,11 @@ dep=$1 version=$2 +verbose=1 +if [ "$3" == "-q" ]; then + verbose=0; +fi + if [ -z "$dep" -o -z "$version" ]; then echo "What dependency?" @@ -38,7 +43,12 @@ if [ $lineNo -lt $dependenciesTagLineNo ]; then exit 0; fi -echo "Going to update version of $dep at line $lineNo" +if [ "$verbose" == "1" ]; then + echo "Going to update version of $dep at line $lineNo" +fi + gsed -i "${lineNo}s/.*$version