From e86d581fe4acf5c363ad41fdf4ff94f3fea8f8d1 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Mar 2024 19:55:38 -0500 Subject: [PATCH] CE-881 - Add QueryHints enum & set to QueryInput; do mysql result set streaming based on the POTENTIALLY_LARGE_NUMBER_OF_RESULTS hint being present --- .../actions/tables/query/QueryInput.java | 79 +++++++++++++++++++ .../rdbms/actions/RDBMSQueryAction.java | 24 +++--- 2 files changed, 94 insertions(+), 9 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java index 0c9c24fb..8b88008e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; import java.util.ArrayList; import java.util.Collection; +import java.util.EnumSet; import java.util.List; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; @@ -68,6 +69,24 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn private boolean includeAssociations = false; private Collection associationNamesToInclude = null; + private EnumSet queryHints = EnumSet.noneOf(QueryHint.class); + + + + /******************************************************************************* + ** Information about the query that an application (or qqq service) may know and + ** want to tell the backend, that can help influence how the backend processes + ** query. + ** + ** For example, a query with potentially a large result set, for MySQL backend, + ** we may want to configure the result set to stream results rather than do its + ** default in-memory thing. See RDBMSQueryAction for usage. + *******************************************************************************/ + public enum QueryHint + { + POTENTIALLY_LARGE_NUMBER_OF_RESULTS + } + /******************************************************************************* @@ -569,4 +588,64 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn return (this); } + + + /******************************************************************************* + ** Getter for queryHints + *******************************************************************************/ + public EnumSet getQueryHints() + { + return (this.queryHints); + } + + + + /******************************************************************************* + ** Setter for queryHints + *******************************************************************************/ + public void setQueryHints(EnumSet queryHints) + { + this.queryHints = queryHints; + } + + + + /******************************************************************************* + ** Fluent setter for queryHints + *******************************************************************************/ + public QueryInput withQueryHints(EnumSet queryHints) + { + this.queryHints = queryHints; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for queryHints + *******************************************************************************/ + public QueryInput withQueryHint(QueryHint queryHint) + { + if(this.queryHints == null) + { + this.queryHints = EnumSet.noneOf(QueryHint.class); + } + this.queryHints.add(queryHint); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for queryHints + *******************************************************************************/ + public QueryInput withoutQueryHint(QueryHint queryHint) + { + if(this.queryHints != null) + { + this.queryHints.remove(queryHint); + } + return (this); + } + } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 698a2053..7e7dbd2c 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -357,16 +357,22 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf *******************************************************************************/ private PreparedStatement createStatement(Connection connection, String sql, QueryInput queryInput) throws SQLException { - if(mysqlResultSetOptimizationEnabled && connection.getClass().getName().startsWith("com.mysql")) + if(connection.getClass().getName().startsWith("com.mysql")) { - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html // - // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows). // - // with this change, we start to get results immediately, and the total runtime also seems lower... // - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - statement.setFetchSize(Integer.MIN_VALUE); - return (statement); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we're allowed to use the mysqlResultSetOptimization, and we have the query hint of "expected large result set", then do it. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(mysqlResultSetOptimizationEnabled && queryInput.getQueryHints() != null && queryInput.getQueryHints().contains(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS)) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html // + // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows). // + // with this change, we start to get results immediately, and the total runtime also seems lower... // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + statement.setFetchSize(Integer.MIN_VALUE); + return (statement); + } } return (connection.prepareStatement(sql));