QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #44 from Kingsrook/hotfix/export-crashes

Browse Source 
Hotfix/export crashes
This commit is contained in:
darin.kelkhoff
2023-10-13 10:37:01 -05:00
committed by GitHub
parent 7ef8f9f181 d905efb1c4
commit 61a6c0e8b0
5 changed files with 230 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java
View File

@ -32,11 +32,14 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QValueException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@ -74,9 +77,47 @@ public class QPossibleValueTranslator
///////////////////////////////////////////////////////
private Map<String, Map<Serializable, String>> possibleValueCache = new HashMap<>();
private int maxSizePerPvsCache = 50_000;
private Map<String, QBackendTransaction> transactionsPerTable = new HashMap<>();
// todo not commit - remove instance & session - use Context
boolean useTransactionsAsConnectionPool = false;
/*******************************************************************************
**
*******************************************************************************/
private QBackendTransaction getTransaction(String tableName)
{
/////////////////////////////////////////////////////////////
// mmm, this does cut down on connections used - //
// especially seems helpful in big exports. //
// but, let's just start using connection pools instead... //
/////////////////////////////////////////////////////////////
if(useTransactionsAsConnectionPool)
{
try
{
if(!transactionsPerTable.containsKey(tableName))
{
transactionsPerTable.put(tableName, new InsertAction().openTransaction(new InsertInput(tableName)));
}
return (transactionsPerTable.get(tableName));
}
catch(Exception e)
{
LOG.warn("Error opening transaction for table", logPair("tableName", tableName));
}
}
return null;
}
/*******************************************************************************
** Constructor
@ -425,9 +466,10 @@ public class QPossibleValueTranslator
for(Map.Entry<String, Map<Serializable, String>> entry : possibleValueCache.entrySet())
{
int size = entry.getValue().size();
if(size > 50_000)
if(size > maxSizePerPvsCache)
{
LOG.info("Found a big PVS cache - clearing it.", logPair("name", entry.getKey()), logPair("size", size));
entry.getValue().clear();
}
}
@ -521,6 +563,7 @@ public class QPossibleValueTranslator
QueryInput queryInput = new QueryInput();
queryInput.setTableName(tableName);
queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, page)));
queryInput.setTransaction(getTransaction(tableName));
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// when querying for possible values, we do want to generate their display values, which makes record labels, which are usually used as PVS labels //
@ -613,4 +656,24 @@ public class QPossibleValueTranslator
return (count < 5);
}
/*******************************************************************************
** Getter for maxSizePerPvsCache
*******************************************************************************/
public int getMaxSizePerPvsCache()
{
return (this.maxSizePerPvsCache);
}
/*******************************************************************************
** Setter for maxSizePerPvsCache
*******************************************************************************/
public void setMaxSizePerPvsCache(int maxSizePerPvsCache)
{
this.maxSizePerPvsCache = maxSizePerPvsCache;
}
}

19
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapter.java
View File

@ -80,8 +80,23 @@ public class QRecordToCsvAdapter
/*******************************************************************************
** todo - kinda weak... can we find this in a CSV lib??
*******************************************************************************/
private String sanitize(String value)
static String sanitize(String value)
{
return (value.replaceAll("\"", "\"\"").replaceAll("\n", " "));
/////////////////////////////////////////////////////////////////////////////////////
// especially in big exports, we see a TON of memory allocated and CPU spent here, //
// if we just blindly replaceAll. So, only do it if needed. //
/////////////////////////////////////////////////////////////////////////////////////
if(value.contains("\""))
{
value = value.replaceAll("\"", "\"\"");
}
if(value.contains("\n"))
{
value = value.replaceAll("\n", " ");
}
return (value);
}
}

68
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java
View File

@ -460,4 +460,72 @@ public class QPossibleValueTranslatorTest extends BaseTest
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testClearingInternalCaches() throws QException
{
QInstance qInstance = QContext.getQInstance();
QTableMetaData personTable = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
QPossibleValueTranslator possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
QFieldMetaData shapeField = qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField("favoriteShapeId");
TestUtils.insertDefaultShapes(qInstance);
TestUtils.insertExtraShapes(qInstance);
List<QRecord> personRecords = List.of(
new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 1),
new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 2),
new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 3),
new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 4),
new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 5)
);
MemoryRecordStore.setCollectStatistics(true);
MemoryRecordStore.resetStatistics();
possibleValueTranslator.primePvsCache(personTable, personRecords, null, null);
assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1));
assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2));
assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3));
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query");
possibleValueTranslator.primePvsCache(personTable, personRecords, null, null);
assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1));
assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2));
assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3));
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should still just have ran just 1 query");
possibleValueTranslator.setMaxSizePerPvsCache(2);
possibleValueTranslator.primePvsCache(personTable, personRecords, null, null);
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Now, should have ran another query");
assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1));
assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2));
assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3));
///////////////////////////
// reset and start again //
///////////////////////////
possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
MemoryRecordStore.resetStatistics();
possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords);
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query");
possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords);
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query");
possibleValueTranslator.setMaxSizePerPvsCache(2);
possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords);
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran another query");
MemoryRecordStore.resetStatistics();
possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords.subList(0, 3));
possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords.subList(3, 5));
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran 2 more queries");
}
}

63
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapterTest.java Normal file
View File

@ -0,0 +1,63 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.adapters;
import com.kingsrook.qqq.backend.core.BaseTest;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for QRecordToCsvAdapter
*******************************************************************************/
class QRecordToCsvAdapterTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSanitize()
{
assertEquals("foo", QRecordToCsvAdapter.sanitize("foo"));
assertEquals("""
Homer ""Jay"" Simpson""", QRecordToCsvAdapter.sanitize("""
Homer "Jay" Simpson"""));
assertEquals("""
one ""quote"" two ""quotes"".""", QRecordToCsvAdapter.sanitize("""
one "quote" two "quotes"."""));
assertEquals("""
new line""", QRecordToCsvAdapter.sanitize("""
new
line"""));
assertEquals("""
end ""quote"" new line""", QRecordToCsvAdapter.sanitize("""
end "quote" new
line"""));
}
}

18
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -1222,6 +1222,24 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static void insertExtraShapes(QInstance qInstance) throws QException
{
List<QRecord> shapeRecords = List.of(
new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 4).withValue("name", "Rectangle"),
new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 5).withValue("name", "Pentagon"),
new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 6).withValue("name", "Hexagon"));
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TABLE_NAME_SHAPE);
insertInput.setRecords(shapeRecords);
new InsertAction().execute(insertInput);
}
/*******************************************************************************
**
*******************************************************************************/