Merged dev into feature/CE-773-cartonization-playground

Add isEnabled method to meta-data producers; Put interface on top of …

qqq-backend-core/pom.xml

@@ -84,7 +84,7 @@
       <dependency>
          <groupId>org.json</groupId>
          <artifactId>json</artifactId>
-         <version>20230618</version>
+         <version>20231013</version>
       </dependency>
       <dependency>
          <groupId>org.apache.commons</groupId>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAssociatedScriptAction.java

@@ -68,6 +68,7 @@ public class RunAssociatedScriptAction
       new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
 
       output.setOutput(executeCodeOutput.getOutput());
+      output.setScriptRevisionId(scriptRevision.getId());
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/MetaDataProducerInterface.java

@@ -0,0 +1,78 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
+
+
+/*******************************************************************************
+ ** Interface for classes that know how to produce meta data objects.  Useful with
+ ** MetaDataProducerHelper, to put point at a package full of these, and populate
+ ** your whole QInstance.
+ **
+ ** See also MetaDataProducer - an implementer of this interface, which actually
+ ** came first, and is fine to extend if producing a meta-data class is all your
+ ** clas means to do (nice and "Single-responsibility principle").
+ **
+ ** But, in some applications you may want to, for example, have one class that
+ ** defines a process step, and also produces the meta-data for that process, so
+ ** your whole process can just be one class - so then just have your step class
+ ** implement this interface.  or, same idea for a QRecordEntity that provides
+ ** its own TableMetaData.
+ *******************************************************************************/
+public interface MetaDataProducerInterface<T extends TopLevelMetaDataInterface>
+{
+   int DEFAULT_SORT_ORDER = 500;
+
+
+   /*******************************************************************************
+    ** Produce the metaData object.  Generally, you don't want to add it to the instance
+    ** yourself - but the instance is there in case you need it to get other metaData.
+    *******************************************************************************/
+   T produce(QInstance qInstance) throws QException;
+
+
+   /*******************************************************************************
+    ** In case this producer needs to run before (or after) others, this method
+    ** can control influence that (e.g., if used by MetaDataProducerHelper).
+    **
+    ** Smaller values run first.
+    *******************************************************************************/
+   default int getSortOrder()
+   {
+      return (DEFAULT_SORT_ORDER);
+   }
+
+
+   /*******************************************************************************
+    ** turn this producer on or off - e.g., maybe based on an env value.
+    **
+    *******************************************************************************/
+   default boolean isEnabled()
+   {
+      return (true);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/scripts/RunAssociatedScriptOutput.java

@@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
 public class RunAssociatedScriptOutput extends AbstractActionOutput
 {
    private Serializable output;
+   private Integer      scriptRevisionId;
 
 
 
@@ -67,4 +68,36 @@ public class RunAssociatedScriptOutput extends AbstractActionOutput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for scriptRevisionId
+    *******************************************************************************/
+   public Integer getScriptRevisionId()
+   {
+      return (this.scriptRevisionId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for scriptRevisionId
+    *******************************************************************************/
+   public void setScriptRevisionId(Integer scriptRevisionId)
+   {
+      this.scriptRevisionId = scriptRevisionId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for scriptRevisionId
+    *******************************************************************************/
+   public RunAssociatedScriptOutput withScriptRevisionId(Integer scriptRevisionId)
+   {
+      this.scriptRevisionId = scriptRevisionId;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java

@@ -22,7 +22,7 @@
 package com.kingsrook.qqq.backend.core.model.metadata;
 
 
-import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
 
 
 /*******************************************************************************
@@ -30,29 +30,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
  ** MetaDataProducerHelper, to put point at a package full of these, and populate
  ** your whole QInstance.
  *******************************************************************************/
-public abstract class MetaDataProducer<T extends TopLevelMetaDataInterface>
+public abstract class MetaDataProducer<T extends TopLevelMetaDataInterface> implements MetaDataProducerInterface<T>
 {
-   public static final int DEFAULT_SORT_ORDER = 500;
-
-
-
-   /*******************************************************************************
-    ** Produce the metaData object.  Generally, you don't want to add it to the instance
-    ** yourself - but the instance is there in case you need it to get other metaData.
-    *******************************************************************************/
-   public abstract T produce(QInstance qInstance) throws QException;
-
-
-
-   /*******************************************************************************
-    ** In case this producer needs to run before (or after) others, this method
-    ** can control influence that (e.g., if used by MetaDataProducerHelper).
-    **
-    ** Smaller values run first.
-    *******************************************************************************/
-   public int getSortOrder()
-   {
-      return (DEFAULT_SORT_ORDER);
-   }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java

@@ -30,6 +30,7 @@ import java.util.Comparator;
 import java.util.List;
 import com.google.common.reflect.ClassPath;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
@@ -44,7 +45,7 @@ public class MetaDataProducerHelper
 
 
    /*******************************************************************************
-    ** Recursively find all classes in the given package, that extend MetaDataProducer,
+    ** Recursively find all classes in the given package, that implement MetaDataProducerInterface
     ** run them, and add their output to the given qInstance.
     **
     ** Note - they'll be sorted by the sortOrder they provide.
@@ -54,8 +55,8 @@ public class MetaDataProducerHelper
       ////////////////////////////////////////////////////////////////////////
       // find all the meta data producer classes in (and under) the package //
       ////////////////////////////////////////////////////////////////////////
-      List<Class<?>>            classesInPackage = getClassesInPackage(packageName);
-      List<MetaDataProducer<?>> producers        = new ArrayList<>();
+      List<Class<?>>                     classesInPackage = getClassesInPackage(packageName);
+      List<MetaDataProducerInterface<?>> producers        = new ArrayList<>();
       for(Class<?> aClass : classesInPackage)
       {
          try
@@ -65,22 +66,29 @@ public class MetaDataProducerHelper
                continue;
             }
 
-            for(Constructor<?> constructor : aClass.getConstructors())
+            if(MetaDataProducerInterface.class.isAssignableFrom(aClass))
             {
-               if(constructor.getParameterCount() == 0)
+               boolean foundValidConstructor = false;
+               for(Constructor<?> constructor : aClass.getConstructors())
                {
-                  Object o = constructor.newInstance();
-                  if(o instanceof MetaDataProducer<?> metaDataProducer)
+                  if(constructor.getParameterCount() == 0)
                   {
-                     producers.add(metaDataProducer);
+                     Object o = constructor.newInstance();
+                     producers.add((MetaDataProducerInterface<?>) o);
+                     foundValidConstructor = true;
+                     break;
                   }
-                  break;
+               }
+
+               if(!foundValidConstructor)
+               {
+                  LOG.warn("Found a class which implements MetaDataProducerInterface, but it does not have a no-arg constructor, so it cannot be used.", logPair("class", aClass.getSimpleName()));
                }
             }
          }
          catch(Exception e)
          {
-            LOG.info("Error adding metaData from producer", e, logPair("producer", aClass.getSimpleName()));
+            LOG.warn("Error evaluating a possible meta-data producer class", e, logPair("class", aClass.getSimpleName()));
          }
       }
 
@@ -89,8 +97,8 @@ public class MetaDataProducerHelper
       // after all other types (as apps often try to get other types from the instance)         //
       ////////////////////////////////////////////////////////////////////////////////////////////
       producers.sort(Comparator
-         .comparing((MetaDataProducer<?> p) -> p.getSortOrder())
-         .thenComparing((MetaDataProducer<?> p) ->
+         .comparing((MetaDataProducerInterface<?> p) -> p.getSortOrder())
+         .thenComparing((MetaDataProducerInterface<?> p) ->
          {
             try
             {
@@ -110,22 +118,29 @@ public class MetaDataProducerHelper
             }
          }));
 
-      //////////////////////////////////////////////////////////////
-      // execute each one, adding their meta data to the instance //
-      //////////////////////////////////////////////////////////////
-      for(MetaDataProducer<?> producer : producers)
+      ///////////////////////////////////////////////////////////////////////////
+      // execute each one (if enabled), adding their meta data to the instance //
+      ///////////////////////////////////////////////////////////////////////////
+      for(MetaDataProducerInterface<?> producer : producers)
       {
-         try
+         if(producer.isEnabled())
          {
-            TopLevelMetaDataInterface metaData = producer.produce(instance);
-            if(metaData != null)
+            try
             {
-               metaData.addSelfToInstance(instance);
+               TopLevelMetaDataInterface metaData = producer.produce(instance);
+               if(metaData != null)
+               {
+                  metaData.addSelfToInstance(instance);
+               }
+            }
+            catch(Exception e)
+            {
+               LOG.warn("error executing metaDataProducer", logPair("producer", producer.getClass().getSimpleName()), e);
             }
          }
-         catch(Exception e)
+         else
          {
-            LOG.warn("error executing metaDataProducer", logPair("producer", producer.getClass().getSimpleName()), e);
+            LOG.debug("Not using producer which is not enabled", logPair("producer", producer.getClass().getSimpleName()));
          }
       }
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelperTest.java

@@ -23,8 +23,14 @@ package com.kingsrook.qqq.backend.core.model.metadata;
 
 
 import java.io.IOException;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.TestAbstractMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.TestDisabledMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.TestImplementsMetaDataProducer;
 import com.kingsrook.qqq.backend.core.model.metadata.producers.TestMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.TestNoInterfacesExtendsObject;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.TestNoValidConstructorMetaDataProducer;
 import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
@@ -43,6 +49,11 @@ class MetaDataProducerHelperTest
       QInstance qInstance = new QInstance();
       MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance, "com.kingsrook.qqq.backend.core.model.metadata.producers");
       assertTrue(qInstance.getTables().containsKey(TestMetaDataProducer.NAME));
+      assertTrue(qInstance.getTables().containsKey(TestImplementsMetaDataProducer.NAME));
+      assertFalse(qInstance.getTables().containsKey(TestNoValidConstructorMetaDataProducer.NAME));
+      assertFalse(qInstance.getTables().containsKey(TestNoInterfacesExtendsObject.NAME));
+      assertFalse(qInstance.getTables().containsKey(TestAbstractMetaDataProducer.NAME));
+      assertFalse(qInstance.getTables().containsKey(TestDisabledMetaDataProducer.NAME));
    }
 
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestAbstractMetaDataProducer.java

@@ -0,0 +1,48 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.producers;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public abstract class TestAbstractMetaDataProducer extends MetaDataProducer<QTableMetaData>
+{
+   public static final String NAME = "TestAbstractMetaDataProducer";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      return new QTableMetaData().withName(NAME);
+   }
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestDisabledMetaDataProducer.java

@@ -0,0 +1,59 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.producers;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class TestDisabledMetaDataProducer implements MetaDataProducerInterface<QTableMetaData>
+{
+   public static final String NAME = "Disabled";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (false);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      return new QTableMetaData().withName(NAME);
+   }
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestImplementsMetaDataProducer.java

@@ -0,0 +1,48 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.producers;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class TestImplementsMetaDataProducer implements MetaDataProducerInterface<QTableMetaData>
+{
+   public static final String NAME = "BuiltByProducerImplementingInterface";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      return new QTableMetaData().withName(NAME);
+   }
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestNoInterfacesExtendsObject.java

@@ -0,0 +1,46 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.producers;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class TestNoInterfacesExtendsObject
+{
+   public static final String NAME = "TestNoInterfacesExtendsObject";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      return new QTableMetaData().withName(NAME);
+   }
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestNoValidConstructorMetaDataProducer.java

@@ -0,0 +1,58 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.producers;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class TestNoValidConstructorMetaDataProducer extends MetaDataProducer<QTableMetaData>
+{
+   public static final String NAME = "NoValidConstructor";
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public TestNoValidConstructorMetaDataProducer(boolean b)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      return new QTableMetaData().withName(NAME);
+   }
+}

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java

@@ -81,6 +81,7 @@ import org.apache.http.HttpEntity;
 import org.apache.http.HttpEntityEnclosingRequest;
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
+import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
@@ -1048,7 +1049,7 @@ public class BaseAPIActionUtil
          //////////////////////////////////////////////////////
          // make sure to use closeable client to avoid leaks //
          //////////////////////////////////////////////////////
-         try(CloseableHttpClient httpClient = HttpClientBuilder.create().build())
+         try(CloseableHttpClient httpClient = buildHttpClient())
          {
             ////////////////////////////////////////////////////////////
             // call utility methods that populate data in the request //
@@ -1153,6 +1154,25 @@ public class BaseAPIActionUtil
 
 
 
+   /*******************************************************************************
+    ** Build the default HttpClient used by the makeRequest method
+    *******************************************************************************/
+   protected CloseableHttpClient buildHttpClient()
+   {
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // do we want this?? .setConnectionManager(new PoolingHttpClientConnectionManager()) //
+      // needs some good scrutiny.                                                         //
+      ///////////////////////////////////////////////////////////////////////////////////////
+      return HttpClientBuilder.create()
+         .setDefaultRequestConfig(RequestConfig.custom()
+            .setConnectTimeout(getConnectionTimeoutMillis())
+            .setConnectionRequestTimeout(getConnectionRequestTimeoutMillis())
+            .setSocketTimeout(getSocketTimeoutMillis()).build())
+         .build();
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -1439,6 +1459,51 @@ public class BaseAPIActionUtil
 
 
 
+   /*******************************************************************************
+    ** For the HttpClientBuilder RequestConfig, specify its ConnectionTimeout. See
+    ** - https://www.baeldung.com/httpclient-timeout
+    ** - https://hc.apache.org/httpcomponents-client-5.1.x/current/httpclient5/apidocs/org/apache/hc/client5/http/config/RequestConfig.Builder.html
+    *******************************************************************************/
+   protected int getConnectionTimeoutMillis()
+   {
+      //////////////
+      // 1 minute //
+      //////////////
+      return (60 * 1000);
+   }
+
+
+
+   /*******************************************************************************
+    ** For the HttpClientBuilder RequestConfig, specify its ConnectionRequestTimeout. See
+    ** - https://www.baeldung.com/httpclient-timeout
+    ** - https://hc.apache.org/httpcomponents-client-5.1.x/current/httpclient5/apidocs/org/apache/hc/client5/http/config/RequestConfig.Builder.html
+    *******************************************************************************/
+   protected int getConnectionRequestTimeoutMillis()
+   {
+      //////////////
+      // 1 minute //
+      //////////////
+      return (60 * 1000);
+   }
+
+
+
+   /*******************************************************************************
+    ** For the HttpClientBuilder RequestConfig, specify its ConnectionRequestTimeout. See
+    ** - https://www.baeldung.com/httpclient-timeout
+    ** - https://hc.apache.org/httpcomponents-client-5.1.x/current/httpclient5/apidocs/org/apache/hc/client5/http/config/RequestConfig.Builder.html
+    *******************************************************************************/
+   protected int getSocketTimeoutMillis()
+   {
+      ///////////////
+      // 3 minutes //
+      ///////////////
+      return (3 * 60 * 1000);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtilTest.java

@@ -22,11 +22,18 @@
 package com.kingsrook.qqq.backend.module.api.actions;
 
 
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
 import java.io.Serializable;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
@@ -36,6 +43,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
@@ -82,6 +90,8 @@ import static org.junit.jupiter.api.Assertions.fail;
  *******************************************************************************/
 class BaseAPIActionUtilTest extends BaseTest
 {
+   private static final QLogger LOG = QLogger.getLogger(BaseAPIActionUtilTest.class);
+
    private static MockApiUtilsHelper mockApiUtilsHelper = new MockApiUtilsHelper();
 
 
@@ -822,6 +832,108 @@ class BaseAPIActionUtilTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTimeouts() throws QException
+   {
+      ShortTimeoutActionUtil shortTimeoutActionUtil = new ShortTimeoutActionUtil();
+      shortTimeoutActionUtil.setBackendMetaData((APIBackendMetaData) QContext.getQInstance().getBackend(TestUtils.MOCK_BACKEND_NAME));
+
+      /////////////////////////////////////////////////////////////
+      // make sure we work correctly with a large enough timeout //
+      /////////////////////////////////////////////////////////////
+      {
+         startSimpleHttpServer(8888);
+         HttpGet request = new HttpGet("http://localhost:8888");
+         shortTimeoutActionUtil.setTimeoutMillis(3000);
+
+         shortTimeoutActionUtil.makeRequest(QContext.getQInstance().getTable(TestUtils.MOCK_TABLE_NAME), request);
+      }
+
+      ////////////////////////////////////////////////
+      // make sure we fail with a too-small timeout //
+      ////////////////////////////////////////////////
+      {
+         startSimpleHttpServer(8889);
+         HttpGet request = new HttpGet("http://localhost:8889");
+         shortTimeoutActionUtil.setTimeoutMillis(1);
+
+         assertThatThrownBy(() -> shortTimeoutActionUtil.makeRequest(QContext.getQInstance().getTable(TestUtils.MOCK_TABLE_NAME), request))
+            .hasRootCauseInstanceOf(SocketTimeoutException.class)
+            .rootCause().hasMessageContaining("timed out");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void startSimpleHttpServer(int port)
+   {
+      Executors.newSingleThreadExecutor().submit(() ->
+      {
+         LOG.info("Listening on " + port);
+         try(ServerSocket serverSocket = new ServerSocket(port))
+         {
+            Socket         clientSocket = serverSocket.accept();
+            PrintWriter    out          = new PrintWriter(clientSocket.getOutputStream(), true);
+            BufferedReader in           = new BufferedReader(new InputStreamReader(clientSocket.getInputStream()));
+            String         greeting     = in.readLine();
+            LOG.info("Read: " + greeting);
+            SleepUtils.sleep(1, TimeUnit.SECONDS);
+            out.println("HTTP/1.1 200 OK");
+            out.close();
+            clientSocket.close();
+         }
+         catch(Exception e)
+         {
+            LOG.info("Exception in simple http server", e);
+         }
+      });
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // give time for the thread w/ the listening socket to start before returning control to the thread that's going to try to connect to it //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      SleepUtils.sleep(100, TimeUnit.MILLISECONDS);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static class ShortTimeoutActionUtil extends BaseAPIActionUtil
+   {
+      private int timeoutMillis = 1;
+
+
+
+      /*******************************************************************************
+       ** Setter for timeoutMillis
+       **
+       *******************************************************************************/
+      public void setTimeoutMillis(int timeoutMillis)
+      {
+         this.timeoutMillis = timeoutMillis;
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      protected int getSocketTimeoutMillis()
+      {
+         return (timeoutMillis);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java

@@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -68,7 +69,17 @@ public abstract class AbstractBaseFilesystemAction<FILE>
    /*******************************************************************************
     ** List the files for a table - to be implemented in module-specific subclasses.
     *******************************************************************************/
-   public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase);
+   public List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase) throws QException
+   {
+      return (listFiles(table, backendBase, null));
+   }
+
+
+
+   /*******************************************************************************
+    ** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses.
+    *******************************************************************************/
+   public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException;
 
    /*******************************************************************************
     ** Read the contents of a file - to be implemented in module-specific subclasses.
@@ -181,6 +192,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
    /*******************************************************************************
     ** Generic implementation of the execute method from the QueryInterface
     *******************************************************************************/
+   @SuppressWarnings("checkstyle:Indentation")
    public QueryOutput executeQuery(QueryInput queryInput) throws QException
    {
       preAction(queryInput.getBackend());
@@ -191,51 +203,76 @@ public abstract class AbstractBaseFilesystemAction<FILE>
 
          QTableMetaData                        table        = queryInput.getTable();
          AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
-         List<FILE>                            files        = listFiles(table, queryInput.getBackend());
+         List<FILE>                            files        = listFiles(table, queryInput.getBackend(), queryInput.getFilter());
 
          for(FILE file : files)
          {
             LOG.info("Processing file: " + getFullPathForFile(file));
-            switch(tableDetails.getRecordFormat())
-            {
-               case CSV:
-               {
-                  String fileContents = IOUtils.toString(readFile(file));
-                  fileContents = customizeFileContentsAfterReading(table, fileContents);
 
-                  if(queryInput.getRecordPipe() != null)
+            InputStream inputStream = readFile(file);
+            switch(tableDetails.getCardinality())
+            {
+               case MANY:
+               {
+                  switch(tableDetails.getRecordFormat())
                   {
-                     new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record ->
+                     case CSV:
                      {
-                        ////////////////////////////////////////////////////////////////////////////////////////////
-                        // Before the records go into the pipe, make sure their backend details are added to them //
-                        ////////////////////////////////////////////////////////////////////////////////////////////
-                        addBackendDetailsToRecord(record, file);
-                     }));
-                  }
-                  else
-                  {
-                     List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
-                     addBackendDetailsToRecords(recordsInFile, file);
-                     queryOutput.addRecords(recordsInFile);
+                        String fileContents = IOUtils.toString(inputStream);
+                        fileContents = customizeFileContentsAfterReading(table, fileContents);
+
+                        if(queryInput.getRecordPipe() != null)
+                        {
+                           new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record ->
+                           {
+                              ////////////////////////////////////////////////////////////////////////////////////////////
+                              // Before the records go into the pipe, make sure their backend details are added to them //
+                              ////////////////////////////////////////////////////////////////////////////////////////////
+                              addBackendDetailsToRecord(record, file);
+                           }));
+                        }
+                        else
+                        {
+                           List<QRecord> recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null);
+                           addBackendDetailsToRecords(recordsInFile, file);
+                           queryOutput.addRecords(recordsInFile);
+                        }
+                        break;
+                     }
+                     case JSON:
+                     {
+                        String fileContents = IOUtils.toString(inputStream);
+                        fileContents = customizeFileContentsAfterReading(table, fileContents);
+
+                        // todo - pipe support!!
+                        List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
+                        addBackendDetailsToRecords(recordsInFile, file);
+
+                        queryOutput.addRecords(recordsInFile);
+                        break;
+                     }
+                     default:
+                     {
+                        throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat());
+                     }
                   }
                   break;
                }
-               case JSON:
+               case ONE:
                {
-                  String fileContents = IOUtils.toString(readFile(file));
-                  fileContents = customizeFileContentsAfterReading(table, fileContents);
+                  String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table);
 
-                  // todo - pipe support!!
-                  List<QRecord> recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null);
-                  addBackendDetailsToRecords(recordsInFile, file);
+                  byte[] bytes = inputStream.readAllBytes();
+                  QRecord record = new QRecord()
+                     .withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase)
+                     .withValue(tableDetails.getContentsFieldName(), bytes);
+                  queryOutput.addRecord(record);
 
-                  queryOutput.addRecords(recordsInFile);
                   break;
                }
                default:
                {
-                  throw new NotImplementedException("Filesystem record format " + tableDetails.getRecordFormat() + " is not yet implemented");
+                  throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality());
                }
             }
          }
@@ -342,8 +379,8 @@ public abstract class AbstractBaseFilesystemAction<FILE>
          {
             for(QRecord record : insertInput.getRecords())
             {
-               String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString("fileName"));
-               writeFile(backend, fullPath, record.getValueByteArray("contents"));
+               String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
+               writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
                record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
                output.addRecord(record);
             }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java

@@ -35,6 +35,11 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
    private RecordFormat recordFormat;
    private Cardinality  cardinality;
 
+   ///////////////////////////////////////////////////////////////////////////////////////////////////
+   // todo default these to null, and give validation error if not set for a cardinality=ONE table? //
+   ///////////////////////////////////////////////////////////////////////////////////////////////////
+   private String contentsFieldName = "contents";
+   private String fileNameFieldName = "fileName";
 
 
    /*******************************************************************************
@@ -175,4 +180,67 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
       return ((T) this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for contentsFieldName
+    *******************************************************************************/
+   public String getContentsFieldName()
+   {
+      return (this.contentsFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for contentsFieldName
+    *******************************************************************************/
+   public void setContentsFieldName(String contentsFieldName)
+   {
+      this.contentsFieldName = contentsFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for contentsFieldName
+    *******************************************************************************/
+   public AbstractFilesystemTableBackendDetails withContentsFieldName(String contentsFieldName)
+   {
+      this.contentsFieldName = contentsFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for fileNameFieldName
+    *******************************************************************************/
+   public String getFileNameFieldName()
+   {
+      return (this.fileNameFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fileNameFieldName
+    *******************************************************************************/
+   public void setFileNameFieldName(String fileNameFieldName)
+   {
+      this.fileNameFieldName = fileNameFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fileNameFieldName
+    *******************************************************************************/
+   public AbstractFilesystemTableBackendDetails withFileNameFieldName(String fileNameFieldName)
+   {
+      this.fileNameFieldName = fileNameFieldName;
+      return (this);
+   }
+
+
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java

@@ -23,19 +23,36 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
 
 
 import java.io.File;
+import java.io.FileFilter;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOCase;
+import org.apache.commons.io.filefilter.AndFileFilter;
+import org.apache.commons.io.filefilter.NameFileFilter;
+import org.apache.commons.io.filefilter.OrFileFilter;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.commons.io.filefilter.WildcardFileFilter;
 
 
 /*******************************************************************************
@@ -51,12 +68,66 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
     ** List the files for this table.
     *******************************************************************************/
    @Override
-   public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase)
+   public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
    {
-      // todo - needs rewritten to do globbing...
       String fullPath  = getFullBasePath(table, backendBase);
       File   directory = new File(fullPath);
-      File[] files     = directory.listFiles();
+      File[] files     = null;
+
+      AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
+
+      FileFilter fileFilter = TrueFileFilter.INSTANCE;
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if each file is its own record (ONE), then we may need to do filtering of the directory listing based on the input filter //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(Cardinality.ONE.equals(tableBackendDetails.getCardinality()))
+      {
+         if(filter != null && filter.hasAnyCriteria())
+         {
+            List<FileFilter> fileFilterList = new ArrayList<>();
+            for(QFilterCriteria criteria : filter.getCriteria())
+            {
+               if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
+               {
+                  if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
+                  {
+                     fileFilterList.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(0))));
+                  }
+                  else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
+                  {
+                     List<NameFileFilter> nameInFilters = new ArrayList<>();
+                     for(int i = 0; i < criteria.getValues().size(); i++)
+                     {
+                        nameInFilters.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(i))));
+                     }
+                     fileFilterList.add(new OrFileFilter(nameInFilters));
+                  }
+                  else
+                  {
+                     throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
+                  }
+               }
+               else
+               {
+                  throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
+               }
+            }
+
+            fileFilter = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? new AndFileFilter(fileFilterList) : new OrFileFilter(fileFilterList);
+         }
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      // if the table has a glob specified, add it as an AND to the filter built to this point //
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      if(StringUtils.hasContent(tableBackendDetails.getGlob()))
+      {
+         WildcardFileFilter globFilenameFilter = new WildcardFileFilter(tableBackendDetails.getGlob(), IOCase.INSENSITIVE);
+         fileFilter = new AndFileFilter(List.of(globFilenameFilter, fileFilter));
+      }
+
+      files = directory.listFiles(fileFilter);
 
       if(files == null)
       {

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncStep.java

@@ -59,31 +59,45 @@ public class FilesystemSyncStep implements BackendStep
     *******************************************************************************/
    @Override
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // defer to a private method here, so we can add a type-parameter for that method to use              //
+      // would think we could do that here, but get compiler error, since this method comes from base class //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////
+      doRun(runBackendStepInput, runBackendStepOutput);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private <F> void doRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
       QTableMetaData sourceTable     = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_SOURCE_TABLE));
       QTableMetaData archiveTable    = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE));
       QTableMetaData processingTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FilesystemSyncProcess.FIELD_PROCESSING_TABLE));
 
-      QBackendMetaData                 sourceBackend    = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName());
-      FilesystemBackendModuleInterface sourceModule     = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
-      AbstractBaseFilesystemAction     sourceActionBase = sourceModule.getActionBase();
+      QBackendMetaData                    sourceBackend    = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName());
+      FilesystemBackendModuleInterface<F> sourceModule     = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
+      AbstractBaseFilesystemAction<F>     sourceActionBase = sourceModule.getActionBase();
       sourceActionBase.preAction(sourceBackend);
-      Map<String, Object> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
+      Map<String, F> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
 
-      QBackendMetaData                 archiveBackend    = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName());
-      FilesystemBackendModuleInterface archiveModule     = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
-      AbstractBaseFilesystemAction     archiveActionBase = archiveModule.getActionBase();
+      QBackendMetaData                    archiveBackend    = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName());
+      FilesystemBackendModuleInterface<F> archiveModule     = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
+      AbstractBaseFilesystemAction<F>     archiveActionBase = archiveModule.getActionBase();
       archiveActionBase.preAction(archiveBackend);
       Set<String> archiveFiles = getFileNames(archiveActionBase, archiveTable, archiveBackend).keySet();
 
-      QBackendMetaData                 processingBackend    = runBackendStepInput.getInstance().getBackendForTable(processingTable.getName());
-      FilesystemBackendModuleInterface processingModule     = (FilesystemBackendModuleInterface) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
-      AbstractBaseFilesystemAction     processingActionBase = processingModule.getActionBase();
+      QBackendMetaData                    processingBackend    = runBackendStepInput.getInstance().getBackendForTable(processingTable.getName());
+      FilesystemBackendModuleInterface<F> processingModule     = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(processingBackend);
+      AbstractBaseFilesystemAction<F>     processingActionBase = processingModule.getActionBase();
       processingActionBase.preAction(processingBackend);
 
       Integer maxFilesToSync  = runBackendStepInput.getValueInteger(FilesystemSyncProcess.FIELD_MAX_FILES_TO_ARCHIVE);
       int     syncedFileCount = 0;
-      for(Map.Entry<String, Object> sourceEntry : sourceFiles.entrySet())
+      for(Map.Entry<String, F> sourceEntry : sourceFiles.entrySet())
       {
          try
          {
@@ -91,20 +105,22 @@ public class FilesystemSyncStep implements BackendStep
             if(!archiveFiles.contains(sourceFileName))
             {
                LOG.info("Syncing file [" + sourceFileName + "] to [" + archiveTable + "] and [" + processingTable + "]");
-               InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue());
-               byte[]      bytes       = inputStream.readAllBytes();
-
-               String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
-               archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
-
-               String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
-               processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
-               syncedFileCount++;
-
-               if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)
+               try(InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue()))
                {
-                  LOG.info("Breaking after syncing " + syncedFileCount + " files");
-                  break;
+                  byte[] bytes = inputStream.readAllBytes();
+
+                  String archivePath = archiveActionBase.getFullBasePath(archiveTable, archiveBackend);
+                  archiveActionBase.writeFile(archiveBackend, archivePath + File.separator + sourceFileName, bytes);
+
+                  String processingPath = processingActionBase.getFullBasePath(processingTable, processingBackend);
+                  processingActionBase.writeFile(processingBackend, processingPath + File.separator + sourceFileName, bytes);
+                  syncedFileCount++;
+
+                  if(maxFilesToSync != null && syncedFileCount >= maxFilesToSync)
+                  {
+                     LOG.info("Breaking after syncing " + syncedFileCount + " files");
+                     break;
+                  }
                }
             }
          }
@@ -120,12 +136,12 @@ public class FilesystemSyncStep implements BackendStep
    /*******************************************************************************
     **
     *******************************************************************************/
-   private Map<String, Object> getFileNames(AbstractBaseFilesystemAction actionBase, QTableMetaData table, QBackendMetaData backend)
+   private <F> Map<String, F> getFileNames(AbstractBaseFilesystemAction<F> actionBase, QTableMetaData table, QBackendMetaData backend) throws QException
    {
-      List<Object>        files = actionBase.listFiles(table, backend);
-      Map<String, Object> rs    = new LinkedHashMap<>();
+      List<F>        files = actionBase.listFiles(table, backend);
+      Map<String, F> rs    = new LinkedHashMap<>();
 
-      for(Object file : files)
+      for(F file : files)
       {
          String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table);
          rs.put(fileName, file);

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java

@@ -30,7 +30,9 @@ import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -126,7 +128,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
     ** List the files for a table.
     *******************************************************************************/
    @Override
-   public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase)
+   public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
    {
       S3BackendMetaData                     s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase);
       AbstractFilesystemTableBackendDetails tableDetails      = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
@@ -138,7 +140,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
       ////////////////////////////////////////////////////////////////////
       // todo - look at metadata to configure the s3 client here?       //
       ////////////////////////////////////////////////////////////////////
-      return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob);
+      return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob, filter, tableDetails);
    }
 
 

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java

@@ -37,7 +37,14 @@ import com.amazonaws.services.s3.model.ListObjectsV2Request;
 import com.amazonaws.services.s3.model.ListObjectsV2Result;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
 
@@ -61,7 +68,19 @@ public class S3Utils
     ** List the objects in an S3 bucket matching a glob, per:
     ** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
     *******************************************************************************/
-   public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob)
+   public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob) throws QException
+   {
+      return listObjectsInBucketMatchingGlob(bucketName, path, glob, null, null);
+   }
+
+
+
+   /*******************************************************************************
+    ** List the objects in an S3 bucket matching a glob, per:
+    ** https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)
+    ** and also - (possibly) apply a file-name filter (based on the table's details).
+    *******************************************************************************/
+   public List<S3ObjectSummary> listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
    {
       //////////////////////////////////////////////////////////////////////////////////////////////////
       // s3 list requests find nothing if the path starts with a /, so strip away any leading slashes //
@@ -77,6 +96,28 @@ public class S3Utils
          prefix = prefix.substring(0, prefix.indexOf('*'));
       }
 
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////
+      // for a file-per-record (ONE) table, we may need to apply the filter to listing.                    //
+      // but for MANY tables, the filtering would be done on the records after they came out of the files. //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////
+      boolean useQQueryFilter = false;
+      if(tableDetails != null && Cardinality.ONE.equals(tableDetails.getCardinality()))
+      {
+         useQQueryFilter = true;
+      }
+
+      if(filter != null && useQQueryFilter)
+      {
+         if(filter.getCriteria() != null && filter.getCriteria().size() == 1)
+         {
+            QFilterCriteria criteria = filter.getCriteria().get(0);
+            if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS))
+            {
+               prefix += "/" + criteria.getValues().get(0);
+            }
+         }
+      }
+
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // mmm, we're assuming here we always want more than 1 file - so there must be some * in the glob.                //
       // That's a bad assumption, as it doesn't consider other wildcards like ? and [-] - but - put that aside for now. //
@@ -86,6 +127,7 @@ public class S3Utils
       {
          glob = "";
       }
+
       if(!glob.contains("*"))
       {
          if(glob.equals(""))
@@ -114,7 +156,7 @@ public class S3Utils
          {
             listObjectsV2Request.setContinuationToken(listObjectsV2Result.getNextContinuationToken());
          }
-         LOG.info("Listing bucket=" + bucketName + ", path=" + path);
+         LOG.info("Listing bucket=" + bucketName + ", path=" + path + ", prefix=" + prefix + ", glob=" + glob);
          listObjectsV2Result = getAmazonS3().listObjectsV2(listObjectsV2Request);
 
          //////////////////////////////////
@@ -149,6 +191,14 @@ public class S3Utils
                continue;
             }
 
+            ///////////////////////////////////////////////////////////////////////////////////
+            // if we're a file-per-record table, and we have a filter, compare the key to it //
+            ///////////////////////////////////////////////////////////////////////////////////
+            if(!doesObjectKeyMatchFilter(key, filter, tableDetails))
+            {
+               continue;
+            }
+
             rs.add(objectSummary);
          }
       }
@@ -159,6 +209,95 @@ public class S3Utils
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private boolean doesObjectKeyMatchFilter(String key, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
+   {
+      if(filter == null || !filter.hasAnyCriteria())
+      {
+         return (true);
+      }
+
+      Path path = Path.of(URI.create("file:///" + key));
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      // foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      for(QFilterCriteria criteria : filter.getCriteria())
+      {
+         boolean matches = doesObjectKeyMatchOneCriteria(criteria, tableDetails, path);
+
+         if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
+         {
+            ////////////////////////////////////////////////////////////////////////////////
+            // if it's not a match, and it's an AND filter, then the whole thing is false //
+            ////////////////////////////////////////////////////////////////////////////////
+            return (false);
+         }
+
+         if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
+         {
+            ////////////////////////////////////////////////////////////
+            // if it's an OR filter, and we've a match, return a true //
+            ////////////////////////////////////////////////////////////
+            return (true);
+         }
+      }
+
+      //////////////////////////////////////////////////////////////////////
+      // if we didn't return above, return now                            //
+      // for an OR - if we didn't find something true, then return false. //
+      // else, an AND - if we didn't find a false, we can return true.    //
+      //////////////////////////////////////////////////////////////////////
+      if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
+      {
+         return (false);
+      }
+
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static boolean doesObjectKeyMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException
+   {
+      if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
+      {
+         if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
+         {
+            return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path));
+         }
+         else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
+         {
+            boolean anyMatch = false;
+            for(int i = 0; i < criteria.getValues().size(); i++)
+            {
+               if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path))
+               {
+                  anyMatch = true;
+                  break;
+               }
+            }
+
+            return (anyMatch);
+         }
+         else
+         {
+            throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
+         }
+      }
+      else
+      {
+         throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Get the contents (as an InputStream) for an object in s3
     *******************************************************************************/
@@ -245,4 +384,5 @@ public class S3Utils
 
       return amazonS3;
    }
+
 }

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java

@@ -25,6 +25,11 @@ package com.kingsrook.qqq.backend.module.filesystem.local;
 import java.io.File;
 import java.io.IOException;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@@ -36,6 +41,8 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
 /*******************************************************************************
@@ -47,6 +54,9 @@ public class FilesystemBackendModuleTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
    @BeforeEach
    public void beforeEach() throws IOException
    {
@@ -55,6 +65,9 @@ public class FilesystemBackendModuleTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
    @AfterEach
    public void afterEach() throws Exception
    {
@@ -63,6 +76,78 @@ public class FilesystemBackendModuleTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testListFiles() throws QException
+   {
+      QInstance      qInstance = TestUtils.defineInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
+
+      AbstractFilesystemAction abstractFilesystemAction = new AbstractFilesystemAction();
+      QBackendMetaData         backend                  = qInstance.getBackendForTable(table.getName());
+
+      //////////////////////////////////////////////////////////
+      // with no filter given, all (3) files should come back //
+      //////////////////////////////////////////////////////////
+      List<File> files = abstractFilesystemAction.listFiles(table, backend);
+      assertEquals(3, files.size());
+
+      /////////////////////////////////////////
+      // filter for a file name that's found //
+      /////////////////////////////////////////
+      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
+      assertEquals(1, files.size());
+      assertEquals("BLOB-2.txt", files.get(0).getName());
+
+      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
+      assertEquals(1, files.size());
+      assertEquals("BLOB-1.txt", files.get(0).getName());
+
+      ///////////////////////////////////
+      // filter for 2 names that exist //
+      ///////////////////////////////////
+      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
+      assertEquals(2, files.size());
+
+      /////////////////////////////////////////////
+      // filter for a file name that isn't found //
+      /////////////////////////////////////////////
+      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
+      assertEquals(0, files.size());
+
+      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
+      assertEquals(1, files.size());
+
+      ////////////////////////////////////////////////////
+      // 2 criteria, and'ed, and can't match, so find 0 //
+      ////////////////////////////////////////////////////
+      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
+         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
+         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
+      assertEquals(0, files.size());
+
+      //////////////////////////////////////////////////
+      // 2 criteria, or'ed, and both match, so find 2 //
+      //////////////////////////////////////////////////
+      files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(
+         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
+         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
+         .withBooleanOperator(QQueryFilter.BooleanOperator.OR));
+      assertEquals(2, files.size());
+
+      //////////////////////////////////////
+      // ensure unsupported filters throw //
+      //////////////////////////////////////
+      assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
+         .hasMessageContaining("Unable to query filesystem table by field");
+      assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
+         .hasMessageContaining("Unable to query filename field using operator");
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java

@@ -89,6 +89,7 @@ public class FilesystemActionTest extends BaseTest
 
       writePersonJSONFiles(baseDirectory);
       writePersonCSVFiles(baseDirectory);
+      writeBlobFiles(baseDirectory);
    }
 
 
@@ -158,6 +159,37 @@ public class FilesystemActionTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    ** Write some data files into the directory for the filesystem module.
+    *******************************************************************************/
+   private void writeBlobFiles(File baseDirectory) throws IOException
+   {
+      String fullPath = baseDirectory.getAbsolutePath();
+      if(TestUtils.defineLocalFilesystemBlobTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)
+      {
+         if(StringUtils.hasContent(details.getBasePath()))
+         {
+            fullPath += File.separatorChar + details.getBasePath();
+         }
+      }
+      fullPath += File.separatorChar;
+
+      String data1 = """
+         Hello, Blob
+         """;
+      FileUtils.writeStringToFile(new File(fullPath + "BLOB-1.txt"), data1);
+
+      String data2 = """
+         Hi Bob""";
+      FileUtils.writeStringToFile(new File(fullPath + "BLOB-2.txt"), data2);
+
+      String data3 = """
+         # Hi MD...""";
+      FileUtils.writeStringToFile(new File(fullPath + "BLOB-3.md"), data3);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java

@@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
 
 
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -32,8 +35,10 @@ import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractPostReadFileCustomizer;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.FilesystemTableCustomizers;
-import org.junit.jupiter.api.Assertions;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
 import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
 /*******************************************************************************
@@ -51,8 +56,8 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
       QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
-      Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream()
+      assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
+      assertTrue(queryOutput.getRecords().stream()
             .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
          "All records should have a full-path in their backend details, matching the test folder name");
    }
@@ -74,14 +79,48 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
 
       queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
       QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
-      Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
-      Assertions.assertTrue(
+      assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
+      assertTrue(
          queryOutput.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")),
          "All records should have their email addresses up-shifted.");
    }
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testQueryForCardinalityOne() throws QException
+   {
+      QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
+      queryInput.setFilter(new QQueryFilter());
+      QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
+      assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
+
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
+      queryOutput = new FilesystemQueryAction().execute(queryInput);
+      assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
+      assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
+
+      ////////////////////////////////////////////////////////////////
+      // put a glob on the table - now should only find 2 txt files //
+      ////////////////////////////////////////////////////////////////
+      QInstance instance = TestUtils.defineInstance();
+      ((FilesystemTableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).getBackendDetails()))
+         .withGlob("*.txt");
+      reInitInstanceInContext(instance);
+
+      queryInput.setFilter(new QQueryFilter());
+      queryOutput = new FilesystemQueryAction().execute(queryInput);
+      assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
    public static class ValueUpshifter extends AbstractPostReadFileCustomizer
    {
       @Override

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java

@@ -26,7 +26,7 @@ import java.util.List;
 import java.util.stream.Collectors;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
-import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@@ -187,7 +187,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QModuleDispatchException
+   private void assertTableListing(S3BackendMetaData backend, QTableMetaData table, String... paths) throws QException
    {
       S3BackendModule  module     = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
       AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();
@@ -207,7 +207,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QModuleDispatchException
+   private void printTableListing(S3BackendMetaData backend, QTableMetaData table) throws QException
    {
       S3BackendModule  module     = (S3BackendModule) new QBackendModuleDispatcher().getQBackendModule(backend);
       AbstractS3Action actionBase = (AbstractS3Action) module.getActionBase();

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java

@@ -62,6 +62,9 @@ public class BaseS3Test extends BaseTest
       amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/2.csv", getCSVData2());
       amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/text.txt", "This is a text test");
       amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/" + SUB_FOLDER + "/3.csv", getCSVData3());
+      amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-1.txt", "Hello, Blob");
+      amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-2.txt", "Hi, Bob");
+      amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-3.md", "# Hi, MD");
    }
 
 

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java

@@ -25,6 +25,11 @@ package com.kingsrook.qqq.backend.module.filesystem.s3;
 import java.util.List;
 import java.util.UUID;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
@@ -32,6 +37,9 @@ import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemExceptio
 import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
 /*******************************************************************************
@@ -43,6 +51,83 @@ public class S3BackendModuleTest extends BaseS3Test
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testListFiles() throws QException
+   {
+      QInstance        qInstance = TestUtils.defineInstance();
+      QTableMetaData   table     = qInstance.getTable(TestUtils.TABLE_NAME_BLOB_S3);
+      QBackendMetaData backend   = qInstance.getBackendForTable(table.getName());
+
+      //////////////////////////////////////////////////////
+      // set up the backend module (e.g., for localstack) //
+      //////////////////////////////////////////////////////
+      S3BackendModule  s3BackendModule = new S3BackendModule();
+      AbstractS3Action actionBase      = (AbstractS3Action) s3BackendModule.getActionBase();
+      actionBase.setS3Utils(getS3Utils());
+
+      //////////////////////////////////////////////////////////
+      // with no filter given, all (3) files should come back //
+      //////////////////////////////////////////////////////////
+      List<S3ObjectSummary> files = actionBase.listFiles(table, backend);
+      assertEquals(3, files.size());
+
+      /////////////////////////////////////////
+      // filter for a file name that's found //
+      /////////////////////////////////////////
+      files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
+      assertEquals(1, files.size());
+      assertThat(files.get(0).getKey()).contains("BLOB-2.txt");
+
+      files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
+      assertEquals(1, files.size());
+      assertThat(files.get(0).getKey()).contains("BLOB-1.txt");
+
+      ///////////////////////////////////
+      // filter for 2 names that exist //
+      ///////////////////////////////////
+      files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt")));
+      assertEquals(2, files.size());
+
+      /////////////////////////////////////////////
+      // filter for a file name that isn't found //
+      /////////////////////////////////////////////
+      files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt")));
+      assertEquals(0, files.size());
+
+      files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt")));
+      assertEquals(1, files.size());
+
+      ////////////////////////////////////////////////////
+      // 2 criteria, and'ed, and can't match, so find 0 //
+      ////////////////////////////////////////////////////
+      files = actionBase.listFiles(table, backend, new QQueryFilter(
+         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
+         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")));
+      assertEquals(0, files.size());
+
+      //////////////////////////////////////////////////
+      // 2 criteria, or'ed, and both match, so find 2 //
+      //////////////////////////////////////////////////
+      files = actionBase.listFiles(table, backend, new QQueryFilter(
+         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"),
+         new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))
+         .withBooleanOperator(QQueryFilter.BooleanOperator.OR));
+      assertEquals(2, files.size());
+
+      //////////////////////////////////////
+      // ensure unsupported filters throw //
+      //////////////////////////////////////
+      assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
+         .hasMessageContaining("Unable to query filesystem table by field");
+      assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
+         .hasMessageContaining("Unable to query filename field using operator");
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java

@@ -23,13 +23,19 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
 
 
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
+import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
 /*******************************************************************************
@@ -66,4 +72,39 @@ public class S3QueryActionTest extends BaseS3Test
       return queryInput;
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testQueryForCardinalityOne() throws QException
+   {
+      QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_S3);
+      queryInput.setFilter(new QQueryFilter());
+
+      S3QueryAction s3QueryAction = new S3QueryAction();
+      s3QueryAction.setS3Utils(getS3Utils());
+
+      QueryOutput queryOutput = s3QueryAction.execute(queryInput);
+      assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
+
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
+      queryOutput = s3QueryAction.execute(queryInput);
+      assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
+      assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
+
+      ////////////////////////////////////////////////////////////////
+      // put a glob on the table - now should only find 2 txt files //
+      ////////////////////////////////////////////////////////////////
+      QInstance instance = TestUtils.defineInstance();
+      ((S3TableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_S3).getBackendDetails()))
+         .withGlob("*.txt");
+      reInitInstanceInContext(instance);
+
+      queryInput.setFilter(new QQueryFilter());
+      queryOutput = s3QueryAction.execute(queryInput);
+      assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
+   }
+
 }

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java

@@ -22,10 +22,10 @@
 package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
 
 
-import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
 import org.apache.commons.io.IOUtils;
 import org.junit.jupiter.api.Test;
@@ -42,14 +42,14 @@ public class S3UtilsTest extends BaseS3Test
     **
     *******************************************************************************/
    @Test
-   public void testListObjectsInBucketAtPath()
+   public void testListObjectsInBucketAtPath() throws QException
    {
       S3Utils s3Utils = getS3Utils();
       assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/").size(), "Expected # of s3 objects without subfolders");
       assertEquals(2, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.csv").size(), "Expected # of csv s3 objects without subfolders");
       assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.txt").size(), "Expected # of txt s3 objects without subfolders");
       assertEquals(0, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/*.pdf").size(), "Expected # of pdf s3 objects without subfolders");
-      assertEquals(4, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
+      assertEquals(7, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, "/**").size(), "Expected # of s3 objects with subfolders");
       assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "/").size(), "With leading slash");
       assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/" + TEST_FOLDER, "").size(), "Without trailing slash");
       assertEquals(3, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//" + TEST_FOLDER, "//").size(), "With multiple leading and trailing slashes");
@@ -60,8 +60,8 @@ public class S3UtilsTest extends BaseS3Test
       assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "").size(), "In the root folder, specified as /");
       assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "//", "").size(), "In the root folder, specified as multiple /s");
       assertEquals(1, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "").size(), "In the root folder, specified as empty-string");
-      assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
-      assertEquals(5, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
+      assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "/", "**").size(), "In the root folder, specified as /, and recursively");
+      assertEquals(8, s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "", "**").size(), "In the root folder, specified as empty-string, and recursively");
    }
 
 
@@ -70,7 +70,7 @@ public class S3UtilsTest extends BaseS3Test
     **
     *******************************************************************************/
    @Test
-   public void testGetObjectAsInputStream() throws IOException
+   public void testGetObjectAsInputStream() throws Exception
    {
       S3Utils               s3Utils           = getS3Utils();
       List<S3ObjectSummary> s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", "");

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java

@@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.ActionTimeoutHelper;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -53,7 +54,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
 import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager;
-import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
 
 
 /*******************************************************************************
@@ -65,6 +65,19 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
    private ActionTimeoutHelper actionTimeoutHelper;
 
+   private static boolean mysqlResultSetOptimizationEnabled = false;
+
+   static
+   {
+      try
+      {
+         mysqlResultSetOptimizationEnabled = new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.rdbms.mysql.resultSetOptimizationEnabled", "QQQ_RDBMS_MYSQL_RESULT_SET_OPTIMIZATION_ENABLED", false);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error reading property/env for mysqlResultSetOptimizationEnabled", e);
+      }
+   }
 
 
    /*******************************************************************************
@@ -343,23 +356,19 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
     *******************************************************************************/
    private PreparedStatement createStatement(Connection connection, String sql, QueryInput queryInput) throws SQLException
    {
-      RDBMSBackendMetaData backend = (RDBMSBackendMetaData) queryInput.getBackend();
-      PreparedStatement    statement;
-      if("mysql".equals(backend.getVendor()))
+      if(mysqlResultSetOptimizationEnabled && connection.getClass().getName().startsWith("com.mysql"))
       {
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-reference-implementation-notes.html //
-         // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows).                     //
-         // with this change, we start to get results immediately, and the total runtime also seems lower...                                                                //
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html //
+         // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows).                 //
+         // with this change, we start to get results immediately, and the total runtime also seems lower...                                                            //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
          statement.setFetchSize(Integer.MIN_VALUE);
+         return (statement);
       }
-      else
-      {
-         statement = connection.prepareStatement(sql);
-      }
-      return (statement);
+
+      return (connection.prepareStatement(sql));
    }