Add c3p0 connection pooling to RDBMS module (ConnectionManager)

qqq-backend-module-rdbms/pom.xml

@@ -50,6 +50,11 @@
          <artifactId>mysql-connector-java</artifactId>
          <version>8.0.30</version>
       </dependency>
+      <dependency>
+         <groupId>com.mchange</groupId>
+         <artifactId>c3p0</artifactId>
+         <version>0.10.0</version>
+      </dependency>
       <dependency>
          <groupId>com.h2database</groupId>
          <artifactId>h2</artifactId>

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.rdbms.actions;
 
 import java.io.Serializable;
 import java.sql.Connection;
+import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -56,6 +57,9 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
       InsertOutput   rs    = new InsertOutput();
       QTableMetaData table = insertInput.getTable();
 
+      Connection connection = null;
+      boolean    needToCloseConnection = false;
+
       try
       {
          List<QFieldMetaData> insertableFields = table.getFields().values().stream()
@@ -72,8 +76,6 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
          List<QRecord> outputRecords = new ArrayList<>();
          rs.setRecords(outputRecords);
 
-         Connection connection;
-         boolean    needToCloseConnection = false;
          if(insertInput.getTransaction() != null && insertInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction)
          {
             connection = rdbmsTransaction.getConnection();
@@ -84,8 +86,6 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
             needToCloseConnection = true;
          }
 
-         try
-         {
          for(List<QRecord> page : CollectionUtils.getPages(insertInput.getRecords(), QueryManager.PAGE_SIZE))
          {
             String        tableName   = escapeIdentifier(getTableName(table));
@@ -158,14 +158,6 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
 
             logSQL(sql, params, mark);
          }
-         }
-         finally
-         {
-            if(needToCloseConnection)
-            {
-               connection.close();
-            }
-         }
 
          return rs;
       }
@@ -173,6 +165,21 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
       {
          throw new QException("Error executing insert: " + e.getMessage(), e);
       }
+      finally
+      {
+         if(needToCloseConnection && connection != null)
+         {
+            try
+            {
+               connection.close();
+            }
+            catch(SQLException se)
+            {
+               LOG.error("Error closing database connection", se);
+            }
+         }
+      }
+
    }
 
 }

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java

@@ -25,8 +25,12 @@ package com.kingsrook.qqq.backend.module.rdbms.jdbc;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
+import com.mchange.v2.c3p0.ComboPooledDataSource;
 
 
 /*******************************************************************************
@@ -34,32 +38,134 @@ import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaDat
  *******************************************************************************/
 public class ConnectionManager
 {
+   private boolean mayUseConnectionPool = true;
+
+   private static Map<String, Boolean>               initedConnectionPool = new HashMap<>();
+   private static Map<String, ComboPooledDataSource> connectionPoolMap    = new HashMap<>();
+
+   private static int usageCounter = 0;
+
 
    /*******************************************************************************
     **
     *******************************************************************************/
    public Connection getConnection(RDBMSBackendMetaData backend) throws SQLException
    {
-      String jdbcURL;
+      usageCounter++;
 
-      if(StringUtils.hasContent(backend.getJdbcUrl()))
+      if(mayUseConnectionPool)
       {
-         jdbcURL = backend.getJdbcUrl();
+         return (getConnectionFromPool(backend));
-      }
-      else
-      {
-         switch(backend.getVendor())
-         {
-            // TODO aws-mysql-jdbc driver not working when running on AWS
-            // jdbcURL = "jdbc:mysql:aws://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=CONVERT_TO_NULL";
-            case "aurora" -> jdbcURL = "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false";
-            case "mysql" -> jdbcURL = "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull";
-            case "h2" -> jdbcURL = "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1";
-            default -> throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor());
-         }
       }
 
+      String jdbcURL = getJdbcUrl(backend);
       return DriverManager.getConnection(jdbcURL, backend.getUsername(), backend.getPassword());
    }
 
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void checkPools()
+   {
+      try
+      {
+         System.out.println("Usages: " + usageCounter);
+
+         for(Map.Entry<String, ComboPooledDataSource> entry : CollectionUtils.nonNullMap(connectionPoolMap).entrySet())
+         {
+            System.out.println("POOL USAGE: " + entry.getKey() + ": " + entry.getValue().getNumBusyConnections());
+            if(entry.getValue().getNumBusyConnections() > 2)
+            {
+               System.out.println("break!");
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         e.printStackTrace();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Connection getConnectionFromPool(RDBMSBackendMetaData backend) throws SQLException
+   {
+      try
+      {
+         if(!initedConnectionPool.getOrDefault(backend.getName(), false))
+         {
+            // todo - some syncrhonized
+            ComboPooledDataSource connectionPool = new ComboPooledDataSource();
+            connectionPool.setDriverClass(getJdbcDriverClassName(backend));
+            connectionPool.setJdbcUrl(getJdbcUrl(backend));
+            connectionPool.setUser(backend.getUsername());
+            connectionPool.setPassword(backend.getPassword());
+
+            connectionPool.setTestConnectionOnCheckout(true);
+
+            //////////////////////////////////////////////////////////////////////////
+            // useful to debug leaking connections - meant for tests only though... //
+            //////////////////////////////////////////////////////////////////////////
+            // connectionPool.setDebugUnreturnedConnectionStackTraces(true);
+            // connectionPool.setUnreturnedConnectionTimeout(10);
+
+            connectionPoolMap.put(backend.getName(), connectionPool);
+            initedConnectionPool.put(backend.getName(), true);
+         }
+
+         return (connectionPoolMap.get(backend.getName()).getConnection());
+      }
+      catch(Exception e)
+      {
+         throw (new SQLException("Error getting connection from pool", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String getJdbcDriverClassName(RDBMSBackendMetaData backend)
+   {
+      if(StringUtils.hasContent(backend.getJdbcDriverClassName()))
+      {
+         return backend.getJdbcDriverClassName();
+      }
+
+      return switch(backend.getVendor())
+      {
+         case "mysql", "aurora" -> "com.mysql.cj.jdbc.Driver";
+         case "h2" -> "org.h2.Driver";
+         default -> throw (new IllegalStateException("We do not know what jdbc driver to use for vendor name [" + backend.getVendor() + "].  Try setting jdbcDriverClassName in your backend meta data."));
+      };
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String getJdbcUrl(RDBMSBackendMetaData backend)
+   {
+      if(StringUtils.hasContent(backend.getJdbcUrl()))
+      {
+         return backend.getJdbcUrl();
+      }
+
+      return switch(backend.getVendor())
+      {
+         // TODO aws-mysql-jdbc driver not working when running on AWS
+         // jdbcURL = "jdbc:mysql:aws://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=CONVERT_TO_NULL";
+         case "aurora" -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false";
+         case "mysql" -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull";
+         case "h2" -> "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1";
+         default -> throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor());
+      };
+   }
+
 }

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java

@@ -40,6 +40,7 @@ public class RDBMSBackendMetaData extends QBackendMetaData
    private String  password;
 
    private String jdbcUrl;
+   private String jdbcDriverClassName;
 
 
 
@@ -314,4 +315,35 @@ public class RDBMSBackendMetaData extends QBackendMetaData
       return (this);
    }
 
+
+   /*******************************************************************************
+    ** Getter for jdbcDriverClassName
+    *******************************************************************************/
+   public String getJdbcDriverClassName()
+   {
+      return (this.jdbcDriverClassName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for jdbcDriverClassName
+    *******************************************************************************/
+   public void setJdbcDriverClassName(String jdbcDriverClassName)
+   {
+      this.jdbcDriverClassName = jdbcDriverClassName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for jdbcDriverClassName
+    *******************************************************************************/
+   public RDBMSBackendMetaData withJdbcDriverClassName(String jdbcDriverClassName)
+   {
+      this.jdbcDriverClassName = jdbcDriverClassName;
+      return (this);
+   }
+
+
 }

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java

@@ -67,5 +67,6 @@ public class RDBMSActionTest extends BaseTest
       ConnectionManager connectionManager = new ConnectionManager();
       Connection        connection        = connectionManager.getConnection(TestUtils.defineBackend());
       QueryManager.executeStatement(connection, sql, resultSetProcessor);
+      connection.close();
    }
 }

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java

@@ -64,7 +64,8 @@ class QueryManagerTest extends BaseTest
    @BeforeEach
    void beforeEach() throws SQLException
    {
-      Connection connection = getConnection();
+      try(Connection connection = getConnection())
+      {
          QueryManager.executeUpdate(connection, """
             CREATE TABLE test_table
             (
@@ -77,6 +78,7 @@ class QueryManagerTest extends BaseTest
             )
             """);
       }
+   }
 
 
 
@@ -86,9 +88,11 @@ class QueryManagerTest extends BaseTest
    @AfterEach
    void afterEach() throws SQLException
    {
-      Connection connection = getConnection();
+      try(Connection connection = getConnection())
+      {
          QueryManager.executeUpdate(connection, "DROP TABLE test_table");
       }
+   }
 
 
 
@@ -108,9 +112,10 @@ class QueryManagerTest extends BaseTest
     *******************************************************************************/
    @Test
    void testBindParams() throws SQLException
+   {
+      try(Connection connection = getConnection())
       {
          long              ctMillis = System.currentTimeMillis();
-      Connection        connection = getConnection();
          PreparedStatement ps       = connection.prepareStatement("UPDATE test_table SET int_col = ? WHERE int_col > 0");
 
          ///////////////////////////////////////////////////////////////////////////////
@@ -160,6 +165,7 @@ class QueryManagerTest extends BaseTest
          // originally longs were being downgraded to int when binding, so, verify that doesn't happen //
          ////////////////////////////////////////////////////////////////////////////////////////////////
       }
+   }
 
 
 
@@ -168,10 +174,11 @@ class QueryManagerTest extends BaseTest
     *******************************************************************************/
    @Test
    void testLongBinding() throws SQLException
+   {
+      try(Connection connection = getConnection())
       {
          Long biggerThanMaxInteger = 2147483648L;
 
-      Connection        connection = getConnection();
          PreparedStatement ps = connection.prepareStatement("INSERT INTO test_table (long_col) VALUES (?)");
          QueryManager.bindParam(ps, 1, biggerThanMaxInteger);
          ps.execute();
@@ -183,6 +190,7 @@ class QueryManagerTest extends BaseTest
          assertTrue(rs.next());
          assertEquals(biggerThanMaxInteger, QueryManager.getLong(rs, "long_col"));
       }
+   }
 
 
 
@@ -191,10 +199,11 @@ class QueryManagerTest extends BaseTest
     *******************************************************************************/
    @Test
    void testGetValueMethods() throws SQLException
+   {
+      try(Connection connection = getConnection())
       {
          Long biggerThanMaxInteger = 2147483648L;
 
-      Connection connection = getConnection();
          QueryManager.executeUpdate(connection, "INSERT INTO test_table (int_col, datetime_col, char_col, long_col) VALUES (1, now(), 'A', " + biggerThanMaxInteger + ")");
          PreparedStatement preparedStatement = connection.prepareStatement("SELECT int_col, datetime_col, char_col, long_col from test_table");
          preparedStatement.execute();
@@ -230,6 +239,7 @@ class QueryManagerTest extends BaseTest
          assertEquals(biggerThanMaxInteger, QueryManager.getLong(rs, "long_col"));
          assertEquals(biggerThanMaxInteger, QueryManager.getLong(rs, 4));
       }
+   }
 
 
 
@@ -239,7 +249,8 @@ class QueryManagerTest extends BaseTest
    @Test
    void testGetValueMethodsReturningNull() throws SQLException
    {
-      Connection connection = getConnection();
+      try(Connection connection = getConnection())
+      {
          QueryManager.executeUpdate(connection, "INSERT INTO test_table (int_col, datetime_col, char_col) VALUES (null, null, null)");
          PreparedStatement preparedStatement = connection.prepareStatement("SELECT * from test_table");
          preparedStatement.execute();
@@ -273,6 +284,7 @@ class QueryManagerTest extends BaseTest
          assertNull(QueryManager.getObject(rs, "char_col"));
          assertNull(QueryManager.getObject(rs, 3));
       }
+   }
 
 
 
@@ -283,7 +295,8 @@ class QueryManagerTest extends BaseTest
    @Test
    void testLocalDate() throws SQLException
    {
-      Connection connection = getConnection();
+      try(Connection connection = getConnection())
+      {
          QueryManager.executeUpdate(connection, "INSERT INTO test_table (date_col) VALUES (?)", LocalDate.of(2013, Month.OCTOBER, 1));
 
          PreparedStatement preparedStatement = connection.prepareStatement("SELECT date_col from test_table");
@@ -315,6 +328,7 @@ class QueryManagerTest extends BaseTest
          assertEquals(0, offsetDateTime.getHour(), "Hour value");
          assertEquals(0, offsetDateTime.getMinute(), "Minute value");
       }
+   }
 
 
 
@@ -324,8 +338,8 @@ class QueryManagerTest extends BaseTest
    @Test
    void testLocalTime() throws SQLException
    {
-      Connection connection = getConnection();
+      try(Connection connection = getConnection())
-
+      {
          ////////////////////////////////////
          // insert one just hour & minutes //
          ////////////////////////////////////
@@ -366,6 +380,7 @@ class QueryManagerTest extends BaseTest
          assertEquals(42, localTime.getMinute(), "Minute value");
          assertEquals(59, localTime.getSecond(), "Second value");
       }
+   }
 
 
 
@@ -375,7 +390,8 @@ class QueryManagerTest extends BaseTest
    @Test
    void testExecuteStatementForSingleValue() throws SQLException
    {
-      Connection connection = getConnection();
+      try(Connection connection = getConnection())
+      {
          QueryManager.executeUpdate(connection, """
             INSERT INTO test_table
             ( int_col, datetime_col, char_col, date_col, time_col )
@@ -397,6 +413,7 @@ class QueryManagerTest extends BaseTest
             """);
          assertEquals(null, QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT int_col FROM test_table WHERE int_col IS NULL"));
       }
+   }
 
 
 
@@ -406,7 +423,8 @@ class QueryManagerTest extends BaseTest
    @Test
    void testQueryForSimpleEntity() throws SQLException
    {
-      Connection connection = getConnection();
+      try(Connection connection = getConnection())
+      {
          QueryManager.executeUpdate(connection, """
             INSERT INTO test_table
             ( int_col, datetime_col, char_col, date_col, time_col )
@@ -418,6 +436,7 @@ class QueryManagerTest extends BaseTest
          assertEquals(47, simpleEntity.get("INT_COL"));
          assertEquals("Q", simpleEntity.get("CHAR_COL"));
       }
+   }
 
 
 
@@ -427,7 +446,8 @@ class QueryManagerTest extends BaseTest
    @Test
    void testQueryForRows() throws SQLException
    {
-      Connection connection = getConnection();
+      try(Connection connection = getConnection())
+      {
          QueryManager.executeUpdate(connection, """
             INSERT INTO test_table
             ( int_col, datetime_col, char_col, date_col, time_col )
@@ -439,5 +459,6 @@ class QueryManagerTest extends BaseTest
          assertEquals(47, rows.get(0).get("INT_COL"));
          assertEquals("Q", rows.get(0).get("CHAR_COL"));
       }
+   }
 
 }