diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java index d797e393..7905e706 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java @@ -32,11 +32,14 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QValueException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -74,9 +77,47 @@ public class QPossibleValueTranslator /////////////////////////////////////////////////////// private Map> possibleValueCache = new HashMap<>(); + private int maxSizePerPvsCache = 50_000; + + private Map transactionsPerTable = new HashMap<>(); + // todo not commit - remove instance & session - use Context + boolean useTransactionsAsConnectionPool = false; + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QBackendTransaction getTransaction(String tableName) + { + ///////////////////////////////////////////////////////////// + // mmm, this does cut down on connections used - // + // especially seems helpful in big exports. // + // but, let's just start using connection pools instead... // + ///////////////////////////////////////////////////////////// + if(useTransactionsAsConnectionPool) + { + try + { + if(!transactionsPerTable.containsKey(tableName)) + { + transactionsPerTable.put(tableName, new InsertAction().openTransaction(new InsertInput(tableName))); + } + + return (transactionsPerTable.get(tableName)); + } + catch(Exception e) + { + LOG.warn("Error opening transaction for table", logPair("tableName", tableName)); + } + } + + return null; + } + /******************************************************************************* ** Constructor @@ -425,9 +466,10 @@ public class QPossibleValueTranslator for(Map.Entry> entry : possibleValueCache.entrySet()) { int size = entry.getValue().size(); - if(size > 50_000) + if(size > maxSizePerPvsCache) { LOG.info("Found a big PVS cache - clearing it.", logPair("name", entry.getKey()), logPair("size", size)); + entry.getValue().clear(); } } @@ -521,6 +563,7 @@ public class QPossibleValueTranslator QueryInput queryInput = new QueryInput(); queryInput.setTableName(tableName); queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, page))); + queryInput.setTransaction(getTransaction(tableName)); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // when querying for possible values, we do want to generate their display values, which makes record labels, which are usually used as PVS labels // @@ -613,4 +656,24 @@ public class QPossibleValueTranslator return (count < 5); } + + + /******************************************************************************* + ** Getter for maxSizePerPvsCache + *******************************************************************************/ + public int getMaxSizePerPvsCache() + { + return (this.maxSizePerPvsCache); + } + + + + /******************************************************************************* + ** Setter for maxSizePerPvsCache + *******************************************************************************/ + public void setMaxSizePerPvsCache(int maxSizePerPvsCache) + { + this.maxSizePerPvsCache = maxSizePerPvsCache; + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapter.java index 7319c779..0018a1aa 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapter.java @@ -80,8 +80,23 @@ public class QRecordToCsvAdapter /******************************************************************************* ** todo - kinda weak... can we find this in a CSV lib?? *******************************************************************************/ - private String sanitize(String value) + static String sanitize(String value) { - return (value.replaceAll("\"", "\"\"").replaceAll("\n", " ")); + ///////////////////////////////////////////////////////////////////////////////////// + // especially in big exports, we see a TON of memory allocated and CPU spent here, // + // if we just blindly replaceAll. So, only do it if needed. // + ///////////////////////////////////////////////////////////////////////////////////// + if(value.contains("\"")) + { + value = value.replaceAll("\"", "\"\""); + } + + if(value.contains("\n")) + { + value = value.replaceAll("\n", " "); + } + + return (value); } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java index 385b512f..5313df6b 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java @@ -460,4 +460,72 @@ public class QPossibleValueTranslatorTest extends BaseTest } } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testClearingInternalCaches() throws QException + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData personTable = qInstance.getTable(TestUtils.TABLE_NAME_PERSON); + QPossibleValueTranslator possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession()); + QFieldMetaData shapeField = qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField("favoriteShapeId"); + + TestUtils.insertDefaultShapes(qInstance); + TestUtils.insertExtraShapes(qInstance); + + List personRecords = List.of( + new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 1), + new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 2), + new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 3), + new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 4), + new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 5) + ); + + MemoryRecordStore.setCollectStatistics(true); + MemoryRecordStore.resetStatistics(); + + possibleValueTranslator.primePvsCache(personTable, personRecords, null, null); + assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1)); + assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2)); + assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3)); + assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query"); + + possibleValueTranslator.primePvsCache(personTable, personRecords, null, null); + assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1)); + assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2)); + assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3)); + assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should still just have ran just 1 query"); + + possibleValueTranslator.setMaxSizePerPvsCache(2); + possibleValueTranslator.primePvsCache(personTable, personRecords, null, null); + + assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Now, should have ran another query"); + + assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1)); + assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2)); + assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3)); + + /////////////////////////// + // reset and start again // + /////////////////////////// + possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession()); + MemoryRecordStore.resetStatistics(); + possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords); + assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query"); + possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords); + assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query"); + + possibleValueTranslator.setMaxSizePerPvsCache(2); + possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords); + assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran another query"); + + MemoryRecordStore.resetStatistics(); + possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords.subList(0, 3)); + possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords.subList(3, 5)); + assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran 2 more queries"); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapterTest.java new file mode 100644 index 00000000..245c6e2e --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapterTest.java @@ -0,0 +1,63 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.adapters; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for QRecordToCsvAdapter + *******************************************************************************/ +class QRecordToCsvAdapterTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSanitize() + { + assertEquals("foo", QRecordToCsvAdapter.sanitize("foo")); + + assertEquals(""" + Homer ""Jay"" Simpson""", QRecordToCsvAdapter.sanitize(""" + Homer "Jay" Simpson""")); + + assertEquals(""" + one ""quote"" two ""quotes"".""", QRecordToCsvAdapter.sanitize(""" + one "quote" two "quotes".""")); + + assertEquals(""" + new line""", QRecordToCsvAdapter.sanitize(""" + new + line""")); + + assertEquals(""" + end ""quote"" new line""", QRecordToCsvAdapter.sanitize(""" + end "quote" new + line""")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index c9c89e2f..12512dfb 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -1222,6 +1222,24 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static void insertExtraShapes(QInstance qInstance) throws QException + { + List shapeRecords = List.of( + new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 4).withValue("name", "Rectangle"), + new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 5).withValue("name", "Pentagon"), + new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 6).withValue("name", "Hexagon")); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TABLE_NAME_SHAPE); + insertInput.setRecords(shapeRecords); + new InsertAction().execute(insertInput); + } + + + /******************************************************************************* ** *******************************************************************************/