Update to *actually* clear the internal cache when it's seen to be big. make the size a param, so test can set it. also add unused method to use transactions as a way to do connection pooling, to avoid so many new connections...

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java

@@ -32,11 +32,14 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QValueException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -74,9 +77,47 @@ public class QPossibleValueTranslator
    ///////////////////////////////////////////////////////
    private Map<String, Map<Serializable, String>> possibleValueCache = new HashMap<>();
 
+   private int maxSizePerPvsCache = 50_000;
+
+   private Map<String, QBackendTransaction> transactionsPerTable = new HashMap<>();
+
    // todo not commit - remove instance & session - use Context
 
 
+   boolean useTransactionsAsConnectionPool = false;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QBackendTransaction getTransaction(String tableName)
+   {
+      /////////////////////////////////////////////////////////////
+      // mmm, this does cut down on connections used -           //
+      // especially seems helpful in big exports.                //
+      // but, let's just start using connection pools instead... //
+      /////////////////////////////////////////////////////////////
+      if(useTransactionsAsConnectionPool)
+      {
+         try
+         {
+            if(!transactionsPerTable.containsKey(tableName))
+            {
+               transactionsPerTable.put(tableName, new InsertAction().openTransaction(new InsertInput(tableName)));
+            }
+
+            return (transactionsPerTable.get(tableName));
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Error opening transaction for table", logPair("tableName", tableName));
+         }
+      }
+
+      return null;
+   }
+
 
    /*******************************************************************************
     ** Constructor
@@ -425,9 +466,10 @@ public class QPossibleValueTranslator
       for(Map.Entry<String, Map<Serializable, String>> entry : possibleValueCache.entrySet())
       {
          int size = entry.getValue().size();
-         if(size > 50_000)
+         if(size > maxSizePerPvsCache)
          {
             LOG.info("Found a big PVS cache - clearing it.", logPair("name", entry.getKey()), logPair("size", size));
+            entry.getValue().clear();
          }
       }
 
@@ -521,6 +563,7 @@ public class QPossibleValueTranslator
             QueryInput queryInput = new QueryInput();
             queryInput.setTableName(tableName);
             queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, page)));
+            queryInput.setTransaction(getTransaction(tableName));
 
             /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             // when querying for possible values, we do want to generate their display values, which makes record labels, which are usually used as PVS labels //
@@ -613,4 +656,24 @@ public class QPossibleValueTranslator
       return (count < 5);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for maxSizePerPvsCache
+    *******************************************************************************/
+   public int getMaxSizePerPvsCache()
+   {
+      return (this.maxSizePerPvsCache);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for maxSizePerPvsCache
+    *******************************************************************************/
+   public void setMaxSizePerPvsCache(int maxSizePerPvsCache)
+   {
+      this.maxSizePerPvsCache = maxSizePerPvsCache;
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java

@@ -460,4 +460,72 @@ public class QPossibleValueTranslatorTest extends BaseTest
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testClearingInternalCaches() throws QException
+   {
+      QInstance                qInstance               = QContext.getQInstance();
+      QTableMetaData           personTable             = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
+      QPossibleValueTranslator possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
+      QFieldMetaData           shapeField              = qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField("favoriteShapeId");
+
+      TestUtils.insertDefaultShapes(qInstance);
+      TestUtils.insertExtraShapes(qInstance);
+
+      List<QRecord> personRecords = List.of(
+         new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 1),
+         new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 2),
+         new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 3),
+         new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 4),
+         new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 5)
+      );
+
+      MemoryRecordStore.setCollectStatistics(true);
+      MemoryRecordStore.resetStatistics();
+
+      possibleValueTranslator.primePvsCache(personTable, personRecords, null, null);
+      assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1));
+      assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2));
+      assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3));
+      assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query");
+
+      possibleValueTranslator.primePvsCache(personTable, personRecords, null, null);
+      assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1));
+      assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2));
+      assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3));
+      assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should still just have ran just 1 query");
+
+      possibleValueTranslator.setMaxSizePerPvsCache(2);
+      possibleValueTranslator.primePvsCache(personTable, personRecords, null, null);
+
+      assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Now, should have ran another query");
+
+      assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1));
+      assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2));
+      assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3));
+
+      ///////////////////////////
+      // reset and start again //
+      ///////////////////////////
+      possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
+      MemoryRecordStore.resetStatistics();
+      possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords);
+      assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query");
+      possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords);
+      assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query");
+
+      possibleValueTranslator.setMaxSizePerPvsCache(2);
+      possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords);
+      assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran another query");
+
+      MemoryRecordStore.resetStatistics();
+      possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords.subList(0, 3));
+      possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords.subList(3, 5));
+      assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran 2 more queries");
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -1222,6 +1222,24 @@ public class TestUtils
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void insertExtraShapes(QInstance qInstance) throws QException
+   {
+      List<QRecord> shapeRecords = List.of(
+         new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 4).withValue("name", "Rectangle"),
+         new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 5).withValue("name", "Pentagon"),
+         new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 6).withValue("name", "Hexagon"));
+
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TABLE_NAME_SHAPE);
+      insertInput.setRecords(shapeRecords);
+      new InsertAction().execute(insertInput);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/