From 2b90d7e4b3c0fb3cc878cf814904841cecf0a34c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 15 Dec 2023 18:36:17 -0600 Subject: [PATCH 1/3] Update to use mysql optimizations for statements on aurora too... --- .../backend/module/rdbms/actions/RDBMSQueryAction.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index fc5dfa79..73dfc776 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -65,6 +65,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf private ActionTimeoutHelper actionTimeoutHelper; + private static boolean loggedMysqlOptimizationsForStatements = false; /******************************************************************************* @@ -345,8 +346,14 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf { RDBMSBackendMetaData backend = (RDBMSBackendMetaData) queryInput.getBackend(); PreparedStatement statement; - if("mysql".equals(backend.getVendor())) + if("mysql".equals(backend.getVendor()) || "aurora".equals(backend.getName())) { + if(!loggedMysqlOptimizationsForStatements) + { + LOG.info("Using mysql optimizations for statements (TYPE_FORWARD_ONLY, CONCUR_READ_ONLY, FetchSize(MIN_VALUE)"); + loggedMysqlOptimizationsForStatements = true; + } + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-reference-implementation-notes.html // // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows). // From 4703d3bb243e1de62d91d048a0252b5fbdf8f6a6 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Sat, 16 Dec 2023 10:27:25 -0600 Subject: [PATCH 2/3] Fixed last commit (meant to use backend.vendor, not name, compare to aurora) --- .../qqq/backend/module/rdbms/actions/RDBMSQueryAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 73dfc776..738ccdd6 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -346,7 +346,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf { RDBMSBackendMetaData backend = (RDBMSBackendMetaData) queryInput.getBackend(); PreparedStatement statement; - if("mysql".equals(backend.getVendor()) || "aurora".equals(backend.getName())) + if("mysql".equals(backend.getVendor()) || "aurora".equals(backend.getVendor())) { if(!loggedMysqlOptimizationsForStatements) { From 5f586d30c776ee768e22c504a71511f51817e755 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Dec 2023 08:45:20 -0600 Subject: [PATCH 3/3] Switch to do mysql optimizations if connection is com.mysql class --- .../rdbms/actions/RDBMSQueryAction.java | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 738ccdd6..36d23362 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -53,7 +53,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; -import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; /******************************************************************************* @@ -65,7 +64,6 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf private ActionTimeoutHelper actionTimeoutHelper; - private static boolean loggedMysqlOptimizationsForStatements = false; /******************************************************************************* @@ -344,21 +342,14 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf *******************************************************************************/ private PreparedStatement createStatement(Connection connection, String sql, QueryInput queryInput) throws SQLException { - RDBMSBackendMetaData backend = (RDBMSBackendMetaData) queryInput.getBackend(); - PreparedStatement statement; - if("mysql".equals(backend.getVendor()) || "aurora".equals(backend.getVendor())) + PreparedStatement statement; + if(connection.getClass().getName().startsWith("com.mysql")) { - if(!loggedMysqlOptimizationsForStatements) - { - LOG.info("Using mysql optimizations for statements (TYPE_FORWARD_ONLY, CONCUR_READ_ONLY, FetchSize(MIN_VALUE)"); - loggedMysqlOptimizationsForStatements = true; - } - - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-reference-implementation-notes.html // - // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows). // - // with this change, we start to get results immediately, and the total runtime also seems lower... // - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html // + // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows). // + // with this change, we start to get results immediately, and the total runtime also seems lower... // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); statement.setFetchSize(Integer.MIN_VALUE); }