QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-19 21:50:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merged hotfix/large-result-query-hint-exports into integration/sprint-40

Browse Source
This commit is contained in:
darin.kelkhoff
2024-04-03 16:50:43 -05:00
parent 9257519462 98fc34fb27
commit e77701ce15
4 changed files with 107 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java
View File

@ -232,6 +232,7 @@ public class ExportAction
}
queryInput.getFilter().setLimit(exportInput.getLimit());
queryInput.setShouldTranslatePossibleValues(true);
queryInput.withQueryHint(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
/////////////////////////////////////////////////////////////////
// tell this query that it needs to put its output into a pipe //

79
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
@ -68,6 +69,24 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
private boolean includeAssociations = false;
private Collection<String> associationNamesToInclude = null;
private EnumSet<QueryHint> queryHints = EnumSet.noneOf(QueryHint.class);
/*******************************************************************************
** Information about the query that an application (or qqq service) may know and
** want to tell the backend, that can help influence how the backend processes
** query.
**
** For example, a query with potentially a large result set, for MySQL backend,
** we may want to configure the result set to stream results rather than do its
** default in-memory thing. See RDBMSQueryAction for usage.
*******************************************************************************/
public enum QueryHint
{
POTENTIALLY_LARGE_NUMBER_OF_RESULTS
}
/*******************************************************************************
@ -569,4 +588,64 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
return (this);
}
/*******************************************************************************
** Getter for queryHints
*******************************************************************************/
public EnumSet<QueryHint> getQueryHints()
{
return (this.queryHints);
}
/*******************************************************************************
** Setter for queryHints
*******************************************************************************/
public void setQueryHints(EnumSet<QueryHint> queryHints)
{
this.queryHints = queryHints;
}
/*******************************************************************************
** Fluent setter for queryHints
*******************************************************************************/
public QueryInput withQueryHints(EnumSet<QueryHint> queryHints)
{
this.queryHints = queryHints;
return (this);
}
/*******************************************************************************
** Fluent setter for queryHints
*******************************************************************************/
public QueryInput withQueryHint(QueryHint queryHint)
{
if(this.queryHints == null)
{
this.queryHints = EnumSet.noneOf(QueryHint.class);
}
this.queryHints.add(queryHint);
return (this);
}
/*******************************************************************************
** Fluent setter for queryHints
*******************************************************************************/
public QueryInput withoutQueryHint(QueryHint queryHint)
{
if(this.queryHints != null)
{
this.queryHints.remove(queryHint);
}
return (this);
}
}

12
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorExtractStep.java
View File

@ -26,6 +26,7 @@ import java.time.Instant;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
@ -57,4 +58,15 @@ public class GarbageCollectorExtractStep extends ExtractViaQueryStep
return super.getQueryFilter(runBackendStepInput);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
protected void customizeInputPreQuery(QueryInput queryInput)
{
queryInput.withQueryHint(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
}
}

24
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java
View File

@ -357,16 +357,22 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
*******************************************************************************/
private PreparedStatement createStatement(Connection connection, String sql, QueryInput queryInput) throws SQLException
{
if(mysqlResultSetOptimizationEnabled && connection.getClass().getName().startsWith("com.mysql"))
if(connection.getClass().getName().startsWith("com.mysql"))
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html //
// without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows). //
// with this change, we start to get results immediately, and the total runtime also seems lower... //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
statement.setFetchSize(Integer.MIN_VALUE);
return (statement);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if we're allowed to use the mysqlResultSetOptimization, and we have the query hint of "expected large result set", then do it. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(mysqlResultSetOptimizationEnabled && queryInput.getQueryHints() != null && queryInput.getQueryHints().contains(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS))
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html //
// without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows). //
// with this change, we start to get results immediately, and the total runtime also seems lower... //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
statement.setFetchSize(Integer.MIN_VALUE);
return (statement);
}
}
return (connection.prepareStatement(sql));