QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update to *actually* clear the internal cache when it's seen to be big. make the size a param, so test can set it. also add unused method to use transactions as a way to do connection pooling, to avoid so many new connections...

Browse Source
This commit is contained in:
darin.kelkhoff
2023-10-12 19:09:12 -05:00
parent 7ef8f9f181
commit 0c2b078af9
3 changed files with 150 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java
View File

@ -32,11 +32,14 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QValueException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@ -74,9 +77,47 @@ public class QPossibleValueTranslator
///////////////////////////////////////////////////////
private Map<String, Map<Serializable, String>> possibleValueCache = new HashMap<>();
private int maxSizePerPvsCache = 50_000;
private Map<String, QBackendTransaction> transactionsPerTable = new HashMap<>();
// todo not commit - remove instance & session - use Context
boolean useTransactionsAsConnectionPool = false;
/*******************************************************************************
**
*******************************************************************************/
private QBackendTransaction getTransaction(String tableName)
{
/////////////////////////////////////////////////////////////
// mmm, this does cut down on connections used - //
// especially seems helpful in big exports. //
// but, let's just start using connection pools instead... //
/////////////////////////////////////////////////////////////
if(useTransactionsAsConnectionPool)
{
try
{
if(!transactionsPerTable.containsKey(tableName))
{
transactionsPerTable.put(tableName, new InsertAction().openTransaction(new InsertInput(tableName)));
}
return (transactionsPerTable.get(tableName));
}
catch(Exception e)
{
LOG.warn("Error opening transaction for table", logPair("tableName", tableName));
}
}
return null;
}
/*******************************************************************************
** Constructor
@ -425,9 +466,10 @@ public class QPossibleValueTranslator
for(Map.Entry<String, Map<Serializable, String>> entry : possibleValueCache.entrySet())
{
int size = entry.getValue().size();
if(size > 50_000)
if(size > maxSizePerPvsCache)
{
LOG.info("Found a big PVS cache - clearing it.", logPair("name", entry.getKey()), logPair("size", size));
entry.getValue().clear();
}
}
@ -521,6 +563,7 @@ public class QPossibleValueTranslator
QueryInput queryInput = new QueryInput();
queryInput.setTableName(tableName);
queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, page)));
queryInput.setTransaction(getTransaction(tableName));
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// when querying for possible values, we do want to generate their display values, which makes record labels, which are usually used as PVS labels //
@ -613,4 +656,24 @@ public class QPossibleValueTranslator
return (count < 5);
}
/*******************************************************************************
** Getter for maxSizePerPvsCache
*******************************************************************************/
public int getMaxSizePerPvsCache()
{
return (this.maxSizePerPvsCache);
}
/*******************************************************************************
** Setter for maxSizePerPvsCache
*******************************************************************************/
public void setMaxSizePerPvsCache(int maxSizePerPvsCache)
{
this.maxSizePerPvsCache = maxSizePerPvsCache;
}
}

68
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java
View File

@ -460,4 +460,72 @@ public class QPossibleValueTranslatorTest extends BaseTest
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testClearingInternalCaches() throws QException
{
QInstance qInstance = QContext.getQInstance();
QTableMetaData personTable = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
QPossibleValueTranslator possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
QFieldMetaData shapeField = qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField("favoriteShapeId");
TestUtils.insertDefaultShapes(qInstance);
TestUtils.insertExtraShapes(qInstance);
List<QRecord> personRecords = List.of(
new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 1),
new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 2),
new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 3),
new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 4),
new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 5)
);
MemoryRecordStore.setCollectStatistics(true);
MemoryRecordStore.resetStatistics();
possibleValueTranslator.primePvsCache(personTable, personRecords, null, null);
assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1));
assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2));
assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3));
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query");
possibleValueTranslator.primePvsCache(personTable, personRecords, null, null);
assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1));
assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2));
assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3));
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should still just have ran just 1 query");
possibleValueTranslator.setMaxSizePerPvsCache(2);
possibleValueTranslator.primePvsCache(personTable, personRecords, null, null);
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Now, should have ran another query");
assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1));
assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2));
assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3));
///////////////////////////
// reset and start again //
///////////////////////////
possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
MemoryRecordStore.resetStatistics();
possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords);
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query");
possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords);
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query");
possibleValueTranslator.setMaxSizePerPvsCache(2);
possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords);
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran another query");
MemoryRecordStore.resetStatistics();
possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords.subList(0, 3));
possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords.subList(3, 5));
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran 2 more queries");
}
}

18
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -1222,6 +1222,24 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static void insertExtraShapes(QInstance qInstance) throws QException
{
List<QRecord> shapeRecords = List.of(
new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 4).withValue("name", "Rectangle"),
new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 5).withValue("name", "Pentagon"),
new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 6).withValue("name", "Hexagon"));
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TABLE_NAME_SHAPE);
insertInput.setRecords(shapeRecords);
new InsertAction().execute(insertInput);
}
/*******************************************************************************
**
*******************************************************************************/