Add ExtractViaBasepullQueryStep; add pagination & piping to api query

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java

@@ -52,7 +52,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
-import com.kingsrook.qqq.backend.core.processes.implementations.general.BasepullConfiguration;
+import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
 import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
 import com.kingsrook.qqq.backend.core.state.StateType;
@@ -69,9 +69,11 @@ import org.apache.logging.log4j.Logger;
  *******************************************************************************/
 public class RunProcessAction
 {
-   private static final Logger LOG                       = LogManager.getLogger(RunProcessAction.class);
+   private static final Logger LOG = LogManager.getLogger(RunProcessAction.class);
-   public static final  String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey";
+
-   public static final  String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey";
+   public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey";
+   public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey";
+   public static final String BASEPULL_TIMESTAMP_FIELD  = "basepullTimestampField";
 
 
 
@@ -422,7 +424,7 @@ public class RunProcessAction
 
 
    /*******************************************************************************
-    **
+    ** Insert or update the last runtime value for this basepull into the backend.
     *******************************************************************************/
    protected void storeLastRunTime(RunProcessInput runProcessInput, QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException
    {
@@ -489,13 +491,22 @@ public class RunProcessAction
 
 
    /*******************************************************************************
-    **
+    ** Lookup the last runtime for this basepull, and set it (plus now) in the process's
+    ** values.
     *******************************************************************************/
    protected void persistLastRunTime(RunProcessInput runProcessInput, QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException
    {
-      ////////////////////////////////////////////////////////////////////////////////////////////////
+      ////////////////////////////////////////////////////////
-      // store 'now', which will be used to update basepull record if process completes sucessfully //
+      // if these values were already computed, don't re-do //
-      ////////////////////////////////////////////////////////////////////////////////////////////////
+      ////////////////////////////////////////////////////////
+      if(runProcessInput.getValue(BASEPULL_THIS_RUNTIME_KEY) != null)
+      {
+         return;
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      // store 'now', which will be used to update basepull record if process completes successfully //
+      /////////////////////////////////////////////////////////////////////////////////////////////////
       Instant now = Instant.now();
       runProcessInput.getValues().put(BASEPULL_THIS_RUNTIME_KEY, now);
 
@@ -533,5 +544,6 @@ public class RunProcessAction
       }
 
       runProcessInput.getValues().put(BASEPULL_LAST_RUNTIME_KEY, lastRunTime);
+      runProcessInput.getValues().put(BASEPULL_TIMESTAMP_FIELD, basepullConfiguration.getTimestampField());
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -316,6 +316,24 @@ public class QInstanceEnricher
       {
          generateAppSections(app);
       }
+
+      for(QAppSection section : CollectionUtils.nonNullList(app.getSections()))
+      {
+         enrichAppSection(section);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void enrichAppSection(QAppSection section)
+   {
+      if(!StringUtils.hasContent(section.getLabel()))
+      {
+         section.setLabel(nameToLabel(section.getName()));
+      }
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java

@@ -419,6 +419,17 @@ public class RunBackendStepInput extends AbstractActionInput
 
 
 
+   /*******************************************************************************
+    ** Getter for a single field's value
+    **
+    *******************************************************************************/
+   public Instant getValueInstant(String fieldName)
+   {
+      return (ValueUtils.getValueAsInstant(getValue(fieldName)));
+   }
+
+
+
    /*******************************************************************************
     ** Accessor for processState - protected, because we generally want to access
     ** its members through wrapper methods, we think

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java

@@ -25,6 +25,9 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
@@ -33,6 +36,8 @@ import java.util.List;
  *******************************************************************************/
 public class QFilterCriteria implements Serializable, Cloneable
 {
+   private static final Logger LOG = LogManager.getLogger(QFilterCriteria.class);
+
    private String             fieldName;
    private QCriteriaOperator  operator;
    private List<Serializable> values;
@@ -183,4 +188,46 @@ public class QFilterCriteria implements Serializable, Cloneable
       this.values = values;
       return this;
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String toString()
+   {
+      StringBuilder rs = new StringBuilder(fieldName);
+      try
+      {
+         rs.append(" ").append(operator).append(" ");
+         if(CollectionUtils.nullSafeHasContents(values))
+         {
+            if(values.size() == 1)
+            {
+               rs.append(values.get(0));
+            }
+            else
+            {
+               int index = 0;
+               for(Serializable value : values)
+               {
+                  if(index++ > 9)
+                  {
+                     rs.append("and ").append(values.size() - index).append(" more");
+                     break;
+                  }
+                  rs.append(value).append(",");
+               }
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error in toString", e);
+         rs.append("Error generating toString...");
+      }
+
+      return (rs.toString());
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterOrderBy.java

@@ -151,4 +151,14 @@ public class QFilterOrderBy implements Serializable, Cloneable
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String toString()
+   {
+      return (fieldName + " " + (isAscending ? "ASC" : "DESC"));
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java

@@ -26,6 +26,8 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
@@ -34,6 +36,8 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
  *******************************************************************************/
 public class QQueryFilter implements Serializable, Cloneable
 {
+   private static final Logger LOG = LogManager.getLogger(QQueryFilter.class);
+
    private List<QFilterCriteria> criteria = new ArrayList<>();
    private List<QFilterOrderBy>  orderBys = new ArrayList<>();
 
@@ -301,4 +305,41 @@ public class QQueryFilter implements Serializable, Cloneable
       subFilters.add(subFilter);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String toString()
+   {
+      StringBuilder rs = new StringBuilder("(");
+      try
+      {
+         for(QFilterCriteria criterion : CollectionUtils.nonNullList(criteria))
+         {
+            rs.append(criterion).append(" ").append(getBooleanOperator());
+         }
+
+         for(QQueryFilter subFilter : CollectionUtils.nonNullList(subFilters))
+         {
+            rs.append(subFilter);
+         }
+         rs.append(")");
+
+         rs.append("OrderBy[");
+         for(QFilterOrderBy orderBy : orderBys)
+         {
+            rs.append(orderBy).append(",");
+         }
+         rs.append("]");
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error in toString", e);
+         rs.append("Error generating toString...");
+      }
+
+      return (rs.toString());
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java

@@ -33,7 +33,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
-import com.kingsrook.qqq.backend.core.processes.implementations.general.BasepullConfiguration;
+import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
 
 
 /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java

@@ -822,4 +822,5 @@ public class QTableMetaData implements QAppChildMetaData, Serializable
 
       return (this);
    }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/BasepullConfiguration.java

@@ -0,0 +1,247 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.basepull;
+
+
+import java.io.Serializable;
+
+
+/*******************************************************************************
+ ** Class for storing all basepull configuration data
+ **
+ *******************************************************************************/
+public class BasepullConfiguration implements Serializable
+{
+   private String tableName; // the table that stores the basepull timestamps
+   private String keyField; // the field in the basepull timestamps table that stores the key of the basepull (e.g., a process name)
+   private String keyValue; // the key applied to the keyField - optional - if not set, process.getName is used.
+
+   private String  lastRunTimeFieldName; // the field in the basepull timestamps table that stores the last-run time for the job.
+   private Integer hoursBackForInitialTimestamp; // for the first-run use-case (where there is no row in the timestamps table), how many hours back in time to look.
+
+   private String timestampField; // the name of the field in the table being queried against the last-run timestamp.
+
+
+
+   /*******************************************************************************
+    ** Getter for tableName
+    **
+    *******************************************************************************/
+   public String getTableName()
+   {
+      return tableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableName
+    **
+    *******************************************************************************/
+   public void setTableName(String tableName)
+   {
+      this.tableName = tableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableName
+    **
+    *******************************************************************************/
+   public BasepullConfiguration withTableName(String tableName)
+   {
+      this.tableName = tableName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for keyField
+    **
+    *******************************************************************************/
+   public String getKeyField()
+   {
+      return keyField;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for keyField
+    **
+    *******************************************************************************/
+   public void setKeyField(String keyField)
+   {
+      this.keyField = keyField;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for keyField
+    **
+    *******************************************************************************/
+   public BasepullConfiguration withKeyField(String keyField)
+   {
+      this.keyField = keyField;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for keyValue
+    **
+    *******************************************************************************/
+   public String getKeyValue()
+   {
+      return keyValue;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for keyValue
+    **
+    *******************************************************************************/
+   public void setKeyValue(String keyValue)
+   {
+      this.keyValue = keyValue;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for keyValue
+    **
+    *******************************************************************************/
+   public BasepullConfiguration withKeyValue(String keyValue)
+   {
+      this.keyValue = keyValue;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for lastRunTimeFieldName
+    **
+    *******************************************************************************/
+   public String getLastRunTimeFieldName()
+   {
+      return lastRunTimeFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for lastRunTimeFieldName
+    **
+    *******************************************************************************/
+   public void setLastRunTimeFieldName(String lastRunTimeFieldName)
+   {
+      this.lastRunTimeFieldName = lastRunTimeFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for lastRunTimeFieldName
+    **
+    *******************************************************************************/
+   public BasepullConfiguration withLastRunTimeFieldName(String lastRunTimeFieldName)
+   {
+      this.lastRunTimeFieldName = lastRunTimeFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for hoursBackForInitialTimestamp
+    **
+    *******************************************************************************/
+   public Integer getHoursBackForInitialTimestamp()
+   {
+      return hoursBackForInitialTimestamp;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for hoursBackForInitialTimestamp
+    **
+    *******************************************************************************/
+   public void setHoursBackForInitialTimestamp(Integer hoursBackForInitialTimestamp)
+   {
+      this.hoursBackForInitialTimestamp = hoursBackForInitialTimestamp;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for hoursBackForInitialTimestamp
+    **
+    *******************************************************************************/
+   public BasepullConfiguration withHoursBackForInitialTimestamp(Integer hoursBackForInitialTimestamp)
+   {
+      this.hoursBackForInitialTimestamp = hoursBackForInitialTimestamp;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for timestampField
+    **
+    *******************************************************************************/
+   public String getTimestampField()
+   {
+      return timestampField;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for timestampField
+    **
+    *******************************************************************************/
+   public void setTimestampField(String timestampField)
+   {
+      this.timestampField = timestampField;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for timestampField
+    **
+    *******************************************************************************/
+   public BasepullConfiguration withTimestampField(String timestampField)
+   {
+      this.timestampField = timestampField;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java

@@ -0,0 +1,82 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.basepull;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
+
+
+/*******************************************************************************
+ ** Version of ExtractViaQueryStep that knows how to set up a basepull query.
+ *******************************************************************************/
+public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected QQueryFilter getQueryFilter(RunBackendStepInput runBackendStepInput) throws QException
+   {
+      //////////////////////////////////////////////////////////////
+      // get input query filter or if not found, create a new one //
+      //////////////////////////////////////////////////////////////
+      QQueryFilter queryFilter = new QQueryFilter();
+      try
+      {
+         queryFilter = super.getQueryFilter(runBackendStepInput);
+      }
+      catch(QException qe)
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////
+         // if we catch here, assume that is because there was no default filter, continue on //
+         ///////////////////////////////////////////////////////////////////////////////////////
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // build up a query filter that is against the source table for the given source table timestamp //
+      // field, finding any records that need processed.                                               //
+      // query will be for:  timestamp > lastRun AND timestamp <= thisRun.                             //
+      // then thisRun will be stored, so the next run shouldn't find any records from thisRun.         //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      queryFilter.addCriteria(new QFilterCriteria()
+         .withFieldName(runBackendStepInput.getValueString(RunProcessAction.BASEPULL_TIMESTAMP_FIELD))
+         .withOperator(QCriteriaOperator.GREATER_THAN)
+         .withValues(List.of(runBackendStepInput.getBasepullLastRunTime())));
+
+      queryFilter.addCriteria(new QFilterCriteria()
+         .withFieldName(runBackendStepInput.getValueString(RunProcessAction.BASEPULL_TIMESTAMP_FIELD))
+         .withOperator(QCriteriaOperator.LESS_THAN_OR_EQUALS)
+         .withValues(List.of(runBackendStepInput.getValueInstant(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY))));
+
+      queryFilter.addOrderBy(new QFilterOrderBy(runBackendStepInput.getValueString(RunProcessAction.BASEPULL_TIMESTAMP_FIELD)));
+
+      return (queryFilter);
+   }
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -680,10 +680,13 @@ class QInstanceValidatorTest
    @Test
    void testAppSectionsMissingLabel()
    {
+      ///////////////////////////////////////////////////////////////////////////////////
+      // the enricher makes a label from the name, so, we'll just make them both null. //
+      ///////////////////////////////////////////////////////////////////////////////////
       QAppMetaData app = new QAppMetaData().withName("test")
          .withChild(new QTableMetaData().withName("test"))
-         .withSection(new QAppSection("Section 1", null, new QIcon("person"), List.of("test"), null, null));
+         .withSection(new QAppSection(null, null, new QIcon("person"), List.of("test"), null, null));
-      assertValidationFailureReasons((qInstance) -> qInstance.addApp(app), "Missing a label");
+      assertValidationFailureReasons((qInstance) -> qInstance.addApp(app), "Missing a label", "Missing a name");
    }
 
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStepTest.java

@@ -0,0 +1,74 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.basepull;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for ExtractViaBasepullQueryStep
+ *******************************************************************************/
+class ExtractViaBasepullQueryStepTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      Instant timestamp = Instant.parse("1980-05-31T15:36:00Z");
+      Instant now       = Instant.now();
+
+      RunBackendStepInput input = new RunBackendStepInput(TestUtils.defineInstance());
+      input.addValue(RunProcessAction.BASEPULL_TIMESTAMP_FIELD, "createDate");
+      input.addValue(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY, now);
+      input.setBasepullLastRunTime(timestamp);
+      QQueryFilter queryFilter = new ExtractViaBasepullQueryStep().getQueryFilter(input);
+
+      System.out.println(queryFilter);
+
+      assertEquals(2, queryFilter.getCriteria().size());
+      assertEquals("createDate", queryFilter.getCriteria().get(0).getFieldName());
+      assertEquals(QCriteriaOperator.GREATER_THAN, queryFilter.getCriteria().get(0).getOperator());
+      assertEquals(timestamp, queryFilter.getCriteria().get(0).getValues().get(0));
+
+      assertEquals("createDate", queryFilter.getCriteria().get(1).getFieldName());
+      assertEquals(QCriteriaOperator.LESS_THAN_OR_EQUALS, queryFilter.getCriteria().get(1).getOperator());
+      assertEquals(now, queryFilter.getCriteria().get(1).getValues().get(0));
+
+      assertEquals(1, queryFilter.getOrderBys().size());
+      assertEquals("createDate", queryFilter.getOrderBys().get(0).getFieldName());
+      assertTrue(queryFilter.getOrderBys().get(0).getIsAscending());
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -87,9 +87,9 @@ import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationM
 import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule;
+import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
-import com.kingsrook.qqq.backend.core.processes.implementations.general.BasepullConfiguration;
 import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIQueryAction.java

@@ -22,13 +22,11 @@
 package com.kingsrook.qqq.backend.module.api.actions;
 
 
-import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
@@ -55,33 +53,83 @@ public class APIQueryAction extends AbstractAPIAction implements QueryInterface
       QTableMetaData table = queryInput.getTable();
       preAction(queryInput);
 
-      try
+      QueryOutput queryOutput   = new QueryOutput(queryInput);
+      Integer     originalLimit = queryInput.getLimit();
+      Integer     limit         = originalLimit;
+      Integer     skip          = queryInput.getSkip();
+
+      if(limit == null)
       {
-         QQueryFilter filter      = queryInput.getFilter();
+         limit = apiActionUtil.getApiStandardLimit();
-         String       paramString = apiActionUtil.buildQueryStringForGet(filter, queryInput.getLimit(), queryInput.getSkip(), table.getFields());
-
-         HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
-         HttpClient        client            = httpClientBuilder.build();
-
-         String url = apiActionUtil.buildTableUrl(table) + paramString;
-         LOG.info("API URL: " + url);
-         HttpGet request = new HttpGet(url);
-
-         apiActionUtil.setupAuthorizationInRequest(request);
-         apiActionUtil.setupContentTypeInRequest(request);
-         apiActionUtil.setupAdditionalHeaders(request);
-
-         HttpResponse  response     = client.execute(request);
-         List<QRecord> queryResults = apiActionUtil.processGetResponse(table, response);
-
-         QueryOutput queryOutput = new QueryOutput(queryInput);
-         queryOutput.addRecords(queryResults);
-         return (queryOutput);
       }
-      catch(Exception e)
+
+      int totalCount = 0;
+      while(true)
       {
-         LOG.warn("Error in API Query", e);
+         try
-         throw new QException("Error executing query: " + e.getMessage(), e);
+         {
+            QQueryFilter filter      = queryInput.getFilter();
+            String       paramString = apiActionUtil.buildQueryStringForGet(filter, limit, skip, table.getFields());
+
+            HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
+            HttpClient        client            = httpClientBuilder.build();
+
+            String url = apiActionUtil.buildTableUrl(table) + paramString;
+            LOG.info("API URL: " + url);
+
+            ///////////////////////////
+            // todo - 429 handling!! //
+            ///////////////////////////
+            HttpGet request = new HttpGet(url);
+
+            apiActionUtil.setupAuthorizationInRequest(request);
+            apiActionUtil.setupContentTypeInRequest(request);
+            apiActionUtil.setupAdditionalHeaders(request);
+
+            HttpResponse response = client.execute(request);
+
+            int count = apiActionUtil.processGetResponse(table, response, queryOutput);
+            totalCount += count;
+
+            /////////////////////////////////////////////////////////////////////////
+            // if we've fetched at least as many as the original limit, then break //
+            /////////////////////////////////////////////////////////////////////////
+            if(originalLimit != null && totalCount >= originalLimit)
+            {
+               return (queryOutput);
+            }
+
+            ////////////////////////////////////////////////////////////////////////////////////
+            // if we got back less than a full page this time, then we must be done, so break //
+            ////////////////////////////////////////////////////////////////////////////////////
+            if(count == 0 || (limit != null && count < limit))
+            {
+               return (queryOutput);
+            }
+
+            ///////////////////////////////////////////////////////////////////
+            // if there's an async callback that says we're cancelled, break //
+            ///////////////////////////////////////////////////////////////////
+            if(queryInput.getAsyncJobCallback().wasCancelRequested())
+            {
+               LOG.info("Breaking query job, as requested.");
+               return (queryOutput);
+            }
+
+            ////////////////////////////////////////////////////////////////////////////
+            // else, increment the skip by the count we just got, and query for more. //
+            ////////////////////////////////////////////////////////////////////////////
+            if(skip == null)
+            {
+               skip = 0;
+            }
+            skip += count;
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Error in API Query", e);
+            throw new QException("Error executing query: " + e.getMessage(), e);
+         }
       }
    }
 

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java

@@ -26,7 +26,6 @@ import java.io.IOException;
 import java.io.Serializable;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
 import java.util.Base64;
 import java.util.List;
 import java.util.Map;
@@ -35,6 +34,8 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -74,7 +75,7 @@ public class BaseAPIActionUtil
     *******************************************************************************/
    public long getMillisToSleepAfterEveryCall()
    {
-      return 0;
+      return (0);
    }
 
 
@@ -84,7 +85,7 @@ public class BaseAPIActionUtil
     *******************************************************************************/
    public int getInitialRateLimitBackoffMillis()
    {
-      return 0;
+      return (0);
    }
 
 
@@ -94,7 +95,17 @@ public class BaseAPIActionUtil
     *******************************************************************************/
    public int getMaxAllowedRateLimitErrors()
    {
-      return 0;
+      return (0);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Integer getApiStandardLimit()
+   {
+      return (20);
    }
 
 
@@ -272,7 +283,7 @@ public class BaseAPIActionUtil
    /*******************************************************************************
     **
     *******************************************************************************/
-   protected List<QRecord> processGetResponse(QTableMetaData table, HttpResponse response) throws IOException
+   protected int processGetResponse(QTableMetaData table, HttpResponse response, QueryOutput queryOutput) throws IOException
    {
       int statusCode = response.getStatusLine().getStatusCode();
       System.out.println(statusCode);
@@ -285,7 +296,7 @@ public class BaseAPIActionUtil
       HttpEntity entity       = response.getEntity();
       String     resultString = EntityUtils.toString(entity);
 
-      List<QRecord> recordList = new ArrayList<>();
+      int count = 0;
       if(StringUtils.hasContent(resultString) && !resultString.equals("null"))
       {
          JSONArray  resultList = null;
@@ -309,16 +320,18 @@ public class BaseAPIActionUtil
          {
             for(int i = 0; i < resultList.length(); i++)
             {
-               recordList.add(jsonObjectToRecord(resultList.getJSONObject(i), table.getFields()));
+               queryOutput.addRecord(jsonObjectToRecord(resultList.getJSONObject(i), table.getFields()));
+               count++;
             }
          }
          else
          {
-            recordList.add(jsonObjectToRecord(jsonObject, table.getFields()));
+            queryOutput.addRecord(jsonObjectToRecord(jsonObject, table.getFields()));
+            count++;
          }
       }
 
-      return (recordList);
+      return (count);
    }
 
 
@@ -483,8 +496,14 @@ public class BaseAPIActionUtil
     *******************************************************************************/
    public Integer processGetResponseForCount(QTableMetaData table, HttpResponse response) throws IOException
    {
-      List<QRecord> queryResults = processGetResponse(table, response);
+      /////////////////////////////////////////////////////////////////////////////////////////
-      return (queryResults.size());
+      // set up a query output with a blank query input - e.g., one that isn't using a pipe. //
+      /////////////////////////////////////////////////////////////////////////////////////////
+      QueryOutput queryOutput = new QueryOutput(new QueryInput());
+      processGetResponse(table, response, queryOutput);
+      List<QRecord> records = queryOutput.getRecords();
+
+      return (records == null ? null : records.size());
    }
 
 }

qqq-dev-tools/bin/reset-snapshot-deps.sh

@@ -9,5 +9,9 @@ CURRENT_VERSION="$(cat $QQQ_DEV_TOOLS_DIR/CURRENT-SNAPSHOT-VERSION)"
 MODULE_LIST_FILE=$QQQ_DEV_TOOLS_DIR/MODULE_LIST
 
 for artifact in $(cat $MODULE_LIST_FILE); do
-   update-dep.sh $artifact ${CURRENT_VERSION}-SNAPSHOT
+   update-dep.sh $artifact ${CURRENT_VERSION}-SNAPSHOT -q
 done
+
+echo
+echo git diff pom.xml
+git diff pom.xml

qqq-dev-tools/bin/update-all-qqq-deps.sh

@@ -9,6 +9,14 @@ CURRENT_VERSION="$(cat $QQQ_DEV_TOOLS_DIR/CURRENT-SNAPSHOT-VERSION)"
 MODULE_LIST_FILE=$QQQ_DEV_TOOLS_DIR/MODULE_LIST
 
 for module in $(cat $MODULE_LIST_FILE); do
+   echo "Updating $module..."
    version=$(get-latest-snapshot.sh $module $CURRENT_VERSION)
-   update-dep.sh $module $version
+   update-dep.sh $module $version -q
 done
+
+echo
+echo git diff pom.xml
+git diff pom.xml
+newVersion=$(grep -A1 qqq-backend-core pom.xml | tail -1 | sed 's/.*<version>//;s/<\/version>.*//;s/-........\......./ snapshot/')
+echo "You might want to commit that with:"
+echo "   git commit -m \"Updated qqq deps to $newVersion\" pom.xml"

qqq-dev-tools/bin/update-dep.sh

@@ -8,6 +8,11 @@
 dep=$1
 version=$2
 
+verbose=1
+if [ "$3" == "-q" ]; then
+   verbose=0;
+fi
+
 if [ -z "$dep" -o -z "$version" ]; then
 
    echo "What dependency?"
@@ -38,7 +43,12 @@ if [ $lineNo -lt $dependenciesTagLineNo ]; then
    exit 0;
 fi
 
-echo "Going to update version of $dep at line $lineNo"
+if [ "$verbose" == "1" ]; then
+   echo "Going to update version of $dep at line $lineNo"
+fi
+
 gsed -i "${lineNo}s/<version>.*</<version>$version</" pom.xml
 
-git diff pom.xml
+if [ "$verbose" == "1" ]; then
+   git diff pom.xml
+fi