Merge pull request #80 from Kingsrook/hotfix/large-result-query-hint-exports

Hotfix/large result query hint exports

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java

@@ -232,6 +232,7 @@ public class ExportAction
       }
       queryInput.getFilter().setLimit(exportInput.getLimit());
       queryInput.setShouldTranslatePossibleValues(true);
+      queryInput.withQueryHint(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
 
       /////////////////////////////////////////////////////////////////
       // tell this query that it needs to put its output into a pipe //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.EnumSet;
 import java.util.List;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
@@ -68,6 +69,24 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
    private boolean            includeAssociations       = false;
    private Collection<String> associationNamesToInclude = null;
 
+   private EnumSet<QueryHint> queryHints = EnumSet.noneOf(QueryHint.class);
+
+
+
+   /*******************************************************************************
+    ** Information about the query that an application (or qqq service) may know and
+    ** want to tell the backend, that can help influence how the backend processes
+    ** query.
+    **
+    ** For example, a query with potentially a large result set, for MySQL backend,
+    ** we may want to configure the result set to stream results rather than do its
+    ** default in-memory thing.  See RDBMSQueryAction for usage.
+    *******************************************************************************/
+   public enum QueryHint
+   {
+      POTENTIALLY_LARGE_NUMBER_OF_RESULTS
+   }
+
 
 
    /*******************************************************************************
@@ -569,4 +588,64 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for queryHints
+    *******************************************************************************/
+   public EnumSet<QueryHint> getQueryHints()
+   {
+      return (this.queryHints);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for queryHints
+    *******************************************************************************/
+   public void setQueryHints(EnumSet<QueryHint> queryHints)
+   {
+      this.queryHints = queryHints;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for queryHints
+    *******************************************************************************/
+   public QueryInput withQueryHints(EnumSet<QueryHint> queryHints)
+   {
+      this.queryHints = queryHints;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for queryHints
+    *******************************************************************************/
+   public QueryInput withQueryHint(QueryHint queryHint)
+   {
+      if(this.queryHints == null)
+      {
+         this.queryHints = EnumSet.noneOf(QueryHint.class);
+      }
+      this.queryHints.add(queryHint);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for queryHints
+    *******************************************************************************/
+   public QueryInput withoutQueryHint(QueryHint queryHint)
+   {
+      if(this.queryHints != null)
+      {
+         this.queryHints.remove(queryHint);
+      }
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorExtractStep.java

@@ -26,6 +26,7 @@ import java.time.Instant;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
@@ -57,4 +58,15 @@ public class GarbageCollectorExtractStep extends ExtractViaQueryStep
       return super.getQueryFilter(runBackendStepInput);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   protected void customizeInputPreQuery(QueryInput queryInput)
+   {
+      queryInput.withQueryHint(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
+   }
+
 }

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java

@@ -357,16 +357,22 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
     *******************************************************************************/
    private PreparedStatement createStatement(Connection connection, String sql, QueryInput queryInput) throws SQLException
    {
-      if(mysqlResultSetOptimizationEnabled && connection.getClass().getName().startsWith("com.mysql"))
+      if(connection.getClass().getName().startsWith("com.mysql"))
       {
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html //
+         // if we're allowed to use the mysqlResultSetOptimization, and we have the query hint of "expected large result set", then do it. //
-         // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows).                 //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // with this change, we start to get results immediately, and the total runtime also seems lower...                                                            //
+         if(mysqlResultSetOptimizationEnabled && queryInput.getQueryHints() != null && queryInput.getQueryHints().contains(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS))
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         {
-         PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         statement.setFetchSize(Integer.MIN_VALUE);
+            // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html //
-         return (statement);
+            // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows).                 //
+            // with this change, we start to get results immediately, and the total runtime also seems lower...                                                            //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+            statement.setFetchSize(Integer.MIN_VALUE);
+            return (statement);
+         }
       }
 
       return (connection.prepareStatement(sql));