QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add c3p0 connection pooling to RDBMS module (ConnectionManager)

Browse Source
This commit is contained in:
darin.kelkhoff
2024-03-12 12:02:36 -05:00
parent 949d9cd088
commit 1062f00ed4
6 changed files with 496 additions and 324 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
qqq-backend-module-rdbms/pom.xml
View File

@ -50,6 +50,11 @@
<artifactId>mysql-connector-java</artifactId>
<version>8.0.30</version>
</dependency>
<dependency>
<groupId>com.mchange</groupId>
<artifactId>c3p0</artifactId>
<version>0.10.0</version>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>

31
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.rdbms.actions;
import java.io.Serializable;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@ -56,6 +57,9 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
InsertOutput rs = new InsertOutput();
QTableMetaData table = insertInput.getTable();
Connection connection = null;
boolean needToCloseConnection = false;
try
{
List<QFieldMetaData> insertableFields = table.getFields().values().stream()
@ -72,8 +76,6 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
List<QRecord> outputRecords = new ArrayList<>();
rs.setRecords(outputRecords);
Connection connection;
boolean needToCloseConnection = false;
if(insertInput.getTransaction() != null && insertInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction)
{
connection = rdbmsTransaction.getConnection();
@ -84,8 +86,6 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
needToCloseConnection = true;
}
try
{
for(List<QRecord> page : CollectionUtils.getPages(insertInput.getRecords(), QueryManager.PAGE_SIZE))
{
String tableName = escapeIdentifier(getTableName(table));
@ -158,14 +158,6 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
logSQL(sql, params, mark);
}
}
finally
{
if(needToCloseConnection)
{
connection.close();
}
}
return rs;
}
@ -173,6 +165,21 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
finally
{
if(needToCloseConnection && connection != null)
{
try
{
connection.close();
}
catch(SQLException se)
{
LOG.error("Error closing database connection", se);
}
}
}
}
}

136
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java
View File

@ -25,8 +25,12 @@ package com.kingsrook.qqq.backend.module.rdbms.jdbc;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import com.mchange.v2.c3p0.ComboPooledDataSource;
/*******************************************************************************
@ -34,32 +38,134 @@ import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaDat
*******************************************************************************/
public class ConnectionManager
{
private boolean mayUseConnectionPool = true;
private static Map<String, Boolean> initedConnectionPool = new HashMap<>();
private static Map<String, ComboPooledDataSource> connectionPoolMap = new HashMap<>();
private static int usageCounter = 0;
/*******************************************************************************
**
*******************************************************************************/
public Connection getConnection(RDBMSBackendMetaData backend) throws SQLException
{
String jdbcURL;
usageCounter++;
if(StringUtils.hasContent(backend.getJdbcUrl()))
if(mayUseConnectionPool)
{
jdbcURL = backend.getJdbcUrl();
}
else
{
switch(backend.getVendor())
{
// TODO aws-mysql-jdbc driver not working when running on AWS
// jdbcURL = "jdbc:mysql:aws://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=CONVERT_TO_NULL";
case "aurora" -> jdbcURL = "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false";
case "mysql" -> jdbcURL = "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull";
case "h2" -> jdbcURL = "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1";
default -> throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor());
}
return (getConnectionFromPool(backend));
}
String jdbcURL = getJdbcUrl(backend);
return DriverManager.getConnection(jdbcURL, backend.getUsername(), backend.getPassword());
}
/*******************************************************************************
**
*******************************************************************************/
public static void checkPools()
{
try
{
System.out.println("Usages: " + usageCounter);
for(Map.Entry<String, ComboPooledDataSource> entry : CollectionUtils.nonNullMap(connectionPoolMap).entrySet())
{
System.out.println("POOL USAGE: " + entry.getKey() + ": " + entry.getValue().getNumBusyConnections());
if(entry.getValue().getNumBusyConnections() > 2)
{
System.out.println("break!");
}
}
}
catch(Exception e)
{
e.printStackTrace();
}
}
/*******************************************************************************
**
*******************************************************************************/
private Connection getConnectionFromPool(RDBMSBackendMetaData backend) throws SQLException
{
try
{
if(!initedConnectionPool.getOrDefault(backend.getName(), false))
{
// todo - some syncrhonized
ComboPooledDataSource connectionPool = new ComboPooledDataSource();
connectionPool.setDriverClass(getJdbcDriverClassName(backend));
connectionPool.setJdbcUrl(getJdbcUrl(backend));
connectionPool.setUser(backend.getUsername());
connectionPool.setPassword(backend.getPassword());
connectionPool.setTestConnectionOnCheckout(true);
//////////////////////////////////////////////////////////////////////////
// useful to debug leaking connections - meant for tests only though... //
//////////////////////////////////////////////////////////////////////////
// connectionPool.setDebugUnreturnedConnectionStackTraces(true);
// connectionPool.setUnreturnedConnectionTimeout(10);
connectionPoolMap.put(backend.getName(), connectionPool);
initedConnectionPool.put(backend.getName(), true);
}
return (connectionPoolMap.get(backend.getName()).getConnection());
}
catch(Exception e)
{
throw (new SQLException("Error getting connection from pool", e));
}
}
/*******************************************************************************
**
*******************************************************************************/
public static String getJdbcDriverClassName(RDBMSBackendMetaData backend)
{
if(StringUtils.hasContent(backend.getJdbcDriverClassName()))
{
return backend.getJdbcDriverClassName();
}
return switch(backend.getVendor())
{
case "mysql", "aurora" -> "com.mysql.cj.jdbc.Driver";
case "h2" -> "org.h2.Driver";
default -> throw (new IllegalStateException("We do not know what jdbc driver to use for vendor name [" + backend.getVendor() + "]. Try setting jdbcDriverClassName in your backend meta data."));
};
}
/*******************************************************************************
**
*******************************************************************************/
public static String getJdbcUrl(RDBMSBackendMetaData backend)
{
if(StringUtils.hasContent(backend.getJdbcUrl()))
{
return backend.getJdbcUrl();
}
return switch(backend.getVendor())
{
// TODO aws-mysql-jdbc driver not working when running on AWS
// jdbcURL = "jdbc:mysql:aws://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=CONVERT_TO_NULL";
case "aurora" -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false";
case "mysql" -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull";
case "h2" -> "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1";
default -> throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor());
};
}
}

32
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java
View File

@ -40,6 +40,7 @@ public class RDBMSBackendMetaData extends QBackendMetaData
private String password;
private String jdbcUrl;
private String jdbcDriverClassName;
@ -314,4 +315,35 @@ public class RDBMSBackendMetaData extends QBackendMetaData
return (this);
}
/*******************************************************************************
** Getter for jdbcDriverClassName
*******************************************************************************/
public String getJdbcDriverClassName()
{
return (this.jdbcDriverClassName);
}
/*******************************************************************************
** Setter for jdbcDriverClassName
*******************************************************************************/
public void setJdbcDriverClassName(String jdbcDriverClassName)
{
this.jdbcDriverClassName = jdbcDriverClassName;
}
/*******************************************************************************
** Fluent setter for jdbcDriverClassName
*******************************************************************************/
public RDBMSBackendMetaData withJdbcDriverClassName(String jdbcDriverClassName)
{
this.jdbcDriverClassName = jdbcDriverClassName;
return (this);
}
}

1
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java
View File

@ -67,5 +67,6 @@ public class RDBMSActionTest extends BaseTest
ConnectionManager connectionManager = new ConnectionManager();
Connection connection = connectionManager.getConnection(TestUtils.defineBackend());
QueryManager.executeStatement(connection, sql, resultSetProcessor);
connection.close();
}
}

45
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java
View File

@ -64,7 +64,8 @@ class QueryManagerTest extends BaseTest
@BeforeEach
void beforeEach() throws SQLException
{
Connection connection = getConnection();
try(Connection connection = getConnection())
{
QueryManager.executeUpdate(connection, """
CREATE TABLE test_table
(
@ -77,6 +78,7 @@ class QueryManagerTest extends BaseTest
)
""");
}
}
@ -86,9 +88,11 @@ class QueryManagerTest extends BaseTest
@AfterEach
void afterEach() throws SQLException
{
Connection connection = getConnection();
try(Connection connection = getConnection())
{
QueryManager.executeUpdate(connection, "DROP TABLE test_table");
}
}
@ -108,9 +112,10 @@ class QueryManagerTest extends BaseTest
*******************************************************************************/
@Test
void testBindParams() throws SQLException
{
try(Connection connection = getConnection())
{
long ctMillis = System.currentTimeMillis();
Connection connection = getConnection();
PreparedStatement ps = connection.prepareStatement("UPDATE test_table SET int_col = ? WHERE int_col > 0");
///////////////////////////////////////////////////////////////////////////////
@ -160,6 +165,7 @@ class QueryManagerTest extends BaseTest
// originally longs were being downgraded to int when binding, so, verify that doesn't happen //
////////////////////////////////////////////////////////////////////////////////////////////////
}
}
@ -168,10 +174,11 @@ class QueryManagerTest extends BaseTest
*******************************************************************************/
@Test
void testLongBinding() throws SQLException
{
try(Connection connection = getConnection())
{
Long biggerThanMaxInteger = 2147483648L;
Connection connection = getConnection();
PreparedStatement ps = connection.prepareStatement("INSERT INTO test_table (long_col) VALUES (?)");
QueryManager.bindParam(ps, 1, biggerThanMaxInteger);
ps.execute();
@ -183,6 +190,7 @@ class QueryManagerTest extends BaseTest
assertTrue(rs.next());
assertEquals(biggerThanMaxInteger, QueryManager.getLong(rs, "long_col"));
}
}
@ -191,10 +199,11 @@ class QueryManagerTest extends BaseTest
*******************************************************************************/
@Test
void testGetValueMethods() throws SQLException
{
try(Connection connection = getConnection())
{
Long biggerThanMaxInteger = 2147483648L;
Connection connection = getConnection();
QueryManager.executeUpdate(connection, "INSERT INTO test_table (int_col, datetime_col, char_col, long_col) VALUES (1, now(), 'A', " + biggerThanMaxInteger + ")");
PreparedStatement preparedStatement = connection.prepareStatement("SELECT int_col, datetime_col, char_col, long_col from test_table");
preparedStatement.execute();
@ -230,6 +239,7 @@ class QueryManagerTest extends BaseTest
assertEquals(biggerThanMaxInteger, QueryManager.getLong(rs, "long_col"));
assertEquals(biggerThanMaxInteger, QueryManager.getLong(rs, 4));
}
}
@ -239,7 +249,8 @@ class QueryManagerTest extends BaseTest
@Test
void testGetValueMethodsReturningNull() throws SQLException
{
Connection connection = getConnection();
try(Connection connection = getConnection())
{
QueryManager.executeUpdate(connection, "INSERT INTO test_table (int_col, datetime_col, char_col) VALUES (null, null, null)");
PreparedStatement preparedStatement = connection.prepareStatement("SELECT * from test_table");
preparedStatement.execute();
@ -273,6 +284,7 @@ class QueryManagerTest extends BaseTest
assertNull(QueryManager.getObject(rs, "char_col"));
assertNull(QueryManager.getObject(rs, 3));
}
}
@ -283,7 +295,8 @@ class QueryManagerTest extends BaseTest
@Test
void testLocalDate() throws SQLException
{
Connection connection = getConnection();
try(Connection connection = getConnection())
{
QueryManager.executeUpdate(connection, "INSERT INTO test_table (date_col) VALUES (?)", LocalDate.of(2013, Month.OCTOBER, 1));
PreparedStatement preparedStatement = connection.prepareStatement("SELECT date_col from test_table");
@ -315,6 +328,7 @@ class QueryManagerTest extends BaseTest
assertEquals(0, offsetDateTime.getHour(), "Hour value");
assertEquals(0, offsetDateTime.getMinute(), "Minute value");
}
}
@ -324,8 +338,8 @@ class QueryManagerTest extends BaseTest
@Test
void testLocalTime() throws SQLException
{
Connection connection = getConnection();
try(Connection connection = getConnection())
{
////////////////////////////////////
// insert one just hour & minutes //
////////////////////////////////////
@ -366,6 +380,7 @@ class QueryManagerTest extends BaseTest
assertEquals(42, localTime.getMinute(), "Minute value");
assertEquals(59, localTime.getSecond(), "Second value");
}
}
@ -375,7 +390,8 @@ class QueryManagerTest extends BaseTest
@Test
void testExecuteStatementForSingleValue() throws SQLException
{
Connection connection = getConnection();
try(Connection connection = getConnection())
{
QueryManager.executeUpdate(connection, """
INSERT INTO test_table
( int_col, datetime_col, char_col, date_col, time_col )
@ -397,6 +413,7 @@ class QueryManagerTest extends BaseTest
""");
assertEquals(null, QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT int_col FROM test_table WHERE int_col IS NULL"));
}
}
@ -406,7 +423,8 @@ class QueryManagerTest extends BaseTest
@Test
void testQueryForSimpleEntity() throws SQLException
{
Connection connection = getConnection();
try(Connection connection = getConnection())
{
QueryManager.executeUpdate(connection, """
INSERT INTO test_table
( int_col, datetime_col, char_col, date_col, time_col )
@ -418,6 +436,7 @@ class QueryManagerTest extends BaseTest
assertEquals(47, simpleEntity.get("INT_COL"));
assertEquals("Q", simpleEntity.get("CHAR_COL"));
}
}
@ -427,7 +446,8 @@ class QueryManagerTest extends BaseTest
@Test
void testQueryForRows() throws SQLException
{
Connection connection = getConnection();
try(Connection connection = getConnection())
{
QueryManager.executeUpdate(connection, """
INSERT INTO test_table
( int_col, datetime_col, char_col, date_col, time_col )
@ -439,5 +459,6 @@ class QueryManagerTest extends BaseTest
assertEquals(47, rows.get(0).get("INT_COL"));
assertEquals("Q", rows.get(0).get("CHAR_COL"));
}
}
}