diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java index 6bd4b83d..5b3fd0d6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java @@ -232,6 +232,7 @@ public class ExportAction } queryInput.getFilter().setLimit(exportInput.getLimit()); queryInput.setShouldTranslatePossibleValues(true); + queryInput.withQueryHint(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS); ///////////////////////////////////////////////////////////////// // tell this query that it needs to put its output into a pipe // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java index 0c9c24fb..8b88008e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; import java.util.ArrayList; import java.util.Collection; +import java.util.EnumSet; import java.util.List; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; @@ -68,6 +69,24 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn private boolean includeAssociations = false; private Collection associationNamesToInclude = null; + private EnumSet queryHints = EnumSet.noneOf(QueryHint.class); + + + + /******************************************************************************* + ** Information about the query that an application (or qqq service) may know and + ** want to tell the backend, that can help influence how the backend processes + ** query. + ** + ** For example, a query with potentially a large result set, for MySQL backend, + ** we may want to configure the result set to stream results rather than do its + ** default in-memory thing. See RDBMSQueryAction for usage. + *******************************************************************************/ + public enum QueryHint + { + POTENTIALLY_LARGE_NUMBER_OF_RESULTS + } + /******************************************************************************* @@ -569,4 +588,64 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn return (this); } + + + /******************************************************************************* + ** Getter for queryHints + *******************************************************************************/ + public EnumSet getQueryHints() + { + return (this.queryHints); + } + + + + /******************************************************************************* + ** Setter for queryHints + *******************************************************************************/ + public void setQueryHints(EnumSet queryHints) + { + this.queryHints = queryHints; + } + + + + /******************************************************************************* + ** Fluent setter for queryHints + *******************************************************************************/ + public QueryInput withQueryHints(EnumSet queryHints) + { + this.queryHints = queryHints; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for queryHints + *******************************************************************************/ + public QueryInput withQueryHint(QueryHint queryHint) + { + if(this.queryHints == null) + { + this.queryHints = EnumSet.noneOf(QueryHint.class); + } + this.queryHints.add(queryHint); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for queryHints + *******************************************************************************/ + public QueryInput withoutQueryHint(QueryHint queryHint) + { + if(this.queryHints != null) + { + this.queryHints.remove(queryHint); + } + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorExtractStep.java index 4c4830d5..3f4ee4c0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorExtractStep.java @@ -26,6 +26,7 @@ import java.time.Instant; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; @@ -57,4 +58,15 @@ public class GarbageCollectorExtractStep extends ExtractViaQueryStep return super.getQueryFilter(runBackendStepInput); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected void customizeInputPreQuery(QueryInput queryInput) + { + queryInput.withQueryHint(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS); + } + } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 698a2053..7e7dbd2c 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -357,16 +357,22 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf *******************************************************************************/ private PreparedStatement createStatement(Connection connection, String sql, QueryInput queryInput) throws SQLException { - if(mysqlResultSetOptimizationEnabled && connection.getClass().getName().startsWith("com.mysql")) + if(connection.getClass().getName().startsWith("com.mysql")) { - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html // - // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows). // - // with this change, we start to get results immediately, and the total runtime also seems lower... // - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - statement.setFetchSize(Integer.MIN_VALUE); - return (statement); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we're allowed to use the mysqlResultSetOptimization, and we have the query hint of "expected large result set", then do it. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(mysqlResultSetOptimizationEnabled && queryInput.getQueryHints() != null && queryInput.getQueryHints().contains(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS)) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html // + // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows). // + // with this change, we start to get results immediately, and the total runtime also seems lower... // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + statement.setFetchSize(Integer.MIN_VALUE); + return (statement); + } } return (connection.prepareStatement(sql));