Merge pull request #44 from Kingsrook/hotfix/export-crashes

Hotfix/export crashes

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java

@@ -32,11 +32,14 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QValueException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -74,9 +77,47 @@ public class QPossibleValueTranslator
    ///////////////////////////////////////////////////////
    private Map<String, Map<Serializable, String>> possibleValueCache = new HashMap<>();
 
+   private int maxSizePerPvsCache = 50_000;
+
+   private Map<String, QBackendTransaction> transactionsPerTable = new HashMap<>();
+
    // todo not commit - remove instance & session - use Context
 
 
+   boolean useTransactionsAsConnectionPool = false;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QBackendTransaction getTransaction(String tableName)
+   {
+      /////////////////////////////////////////////////////////////
+      // mmm, this does cut down on connections used -           //
+      // especially seems helpful in big exports.                //
+      // but, let's just start using connection pools instead... //
+      /////////////////////////////////////////////////////////////
+      if(useTransactionsAsConnectionPool)
+      {
+         try
+         {
+            if(!transactionsPerTable.containsKey(tableName))
+            {
+               transactionsPerTable.put(tableName, new InsertAction().openTransaction(new InsertInput(tableName)));
+            }
+
+            return (transactionsPerTable.get(tableName));
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Error opening transaction for table", logPair("tableName", tableName));
+         }
+      }
+
+      return null;
+   }
+
 
    /*******************************************************************************
     ** Constructor
@@ -425,9 +466,10 @@ public class QPossibleValueTranslator
       for(Map.Entry<String, Map<Serializable, String>> entry : possibleValueCache.entrySet())
       {
          int size = entry.getValue().size();
-         if(size > 50_000)
+         if(size > maxSizePerPvsCache)
          {
             LOG.info("Found a big PVS cache - clearing it.", logPair("name", entry.getKey()), logPair("size", size));
+            entry.getValue().clear();
          }
       }
 
@@ -521,6 +563,7 @@ public class QPossibleValueTranslator
             QueryInput queryInput = new QueryInput();
             queryInput.setTableName(tableName);
             queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, page)));
+            queryInput.setTransaction(getTransaction(tableName));
 
             /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             // when querying for possible values, we do want to generate their display values, which makes record labels, which are usually used as PVS labels //
@@ -613,4 +656,24 @@ public class QPossibleValueTranslator
       return (count < 5);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for maxSizePerPvsCache
+    *******************************************************************************/
+   public int getMaxSizePerPvsCache()
+   {
+      return (this.maxSizePerPvsCache);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for maxSizePerPvsCache
+    *******************************************************************************/
+   public void setMaxSizePerPvsCache(int maxSizePerPvsCache)
+   {
+      this.maxSizePerPvsCache = maxSizePerPvsCache;
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapter.java

@@ -80,8 +80,23 @@ public class QRecordToCsvAdapter
    /*******************************************************************************
     ** todo - kinda weak... can we find this in a CSV lib??
     *******************************************************************************/
-   private String sanitize(String value)
+   static String sanitize(String value)
    {
-      return (value.replaceAll("\"", "\"\"").replaceAll("\n", " "));
+      /////////////////////////////////////////////////////////////////////////////////////
+      // especially in big exports, we see a TON of memory allocated and CPU spent here, //
+      // if we just blindly replaceAll.  So, only do it if needed.                       //
+      /////////////////////////////////////////////////////////////////////////////////////
+      if(value.contains("\""))
+      {
+         value = value.replaceAll("\"", "\"\"");
+      }
+
+      if(value.contains("\n"))
+      {
+         value = value.replaceAll("\n", " ");
+      }
+
+      return (value);
    }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java

@@ -460,4 +460,72 @@ public class QPossibleValueTranslatorTest extends BaseTest
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testClearingInternalCaches() throws QException
+   {
+      QInstance                qInstance               = QContext.getQInstance();
+      QTableMetaData           personTable             = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
+      QPossibleValueTranslator possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
+      QFieldMetaData           shapeField              = qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField("favoriteShapeId");
+
+      TestUtils.insertDefaultShapes(qInstance);
+      TestUtils.insertExtraShapes(qInstance);
+
+      List<QRecord> personRecords = List.of(
+         new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 1),
+         new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 2),
+         new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 3),
+         new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 4),
+         new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 5)
+      );
+
+      MemoryRecordStore.setCollectStatistics(true);
+      MemoryRecordStore.resetStatistics();
+
+      possibleValueTranslator.primePvsCache(personTable, personRecords, null, null);
+      assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1));
+      assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2));
+      assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3));
+      assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query");
+
+      possibleValueTranslator.primePvsCache(personTable, personRecords, null, null);
+      assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1));
+      assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2));
+      assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3));
+      assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should still just have ran just 1 query");
+
+      possibleValueTranslator.setMaxSizePerPvsCache(2);
+      possibleValueTranslator.primePvsCache(personTable, personRecords, null, null);
+
+      assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Now, should have ran another query");
+
+      assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1));
+      assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2));
+      assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3));
+
+      ///////////////////////////
+      // reset and start again //
+      ///////////////////////////
+      possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
+      MemoryRecordStore.resetStatistics();
+      possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords);
+      assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query");
+      possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords);
+      assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query");
+
+      possibleValueTranslator.setMaxSizePerPvsCache(2);
+      possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords);
+      assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran another query");
+
+      MemoryRecordStore.resetStatistics();
+      possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords.subList(0, 3));
+      possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords.subList(3, 5));
+      assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran 2 more queries");
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapterTest.java

@@ -0,0 +1,63 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.adapters;
+
+
+import com.kingsrook.qqq.backend.core.BaseTest;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for QRecordToCsvAdapter 
+ *******************************************************************************/
+class QRecordToCsvAdapterTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSanitize()
+   {
+      assertEquals("foo", QRecordToCsvAdapter.sanitize("foo"));
+
+      assertEquals("""
+         Homer ""Jay"" Simpson""", QRecordToCsvAdapter.sanitize("""
+         Homer "Jay" Simpson"""));
+
+      assertEquals("""
+         one ""quote"" two ""quotes"".""", QRecordToCsvAdapter.sanitize("""
+         one "quote" two "quotes"."""));
+
+      assertEquals("""
+         new line""", QRecordToCsvAdapter.sanitize("""
+         new
+         line"""));
+
+      assertEquals("""
+         end ""quote"" new line""", QRecordToCsvAdapter.sanitize("""
+         end "quote" new
+         line"""));
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -1222,6 +1222,24 @@ public class TestUtils
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void insertExtraShapes(QInstance qInstance) throws QException
+   {
+      List<QRecord> shapeRecords = List.of(
+         new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 4).withValue("name", "Rectangle"),
+         new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 5).withValue("name", "Pentagon"),
+         new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 6).withValue("name", "Hexagon"));
+
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TABLE_NAME_SHAPE);
+      insertInput.setRecords(shapeRecords);
+      new InsertAction().execute(insertInput);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/