QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ability to disable one-off lookups

Browse Source
This commit is contained in:
darin.kelkhoff
2023-02-08 17:02:06 -06:00
parent e3c4a3d91d
commit b0cca3f1d7
3 changed files with 79 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java
View File

@ -301,7 +301,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
*******************************************************************************/
protected void initializeRecordLookupHelper(RunBackendStepInput runBackendStepInput, List<QRecord> sourceRecordList) throws QException
{
this.recordLookupHelper = new RecordLookupHelper(runBackendStepInput);
this.recordLookupHelper = new RecordLookupHelper();
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if there's only 1 record, don't bother preloading all records - just do the single lookup by the single needed key. //

45
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java
View File

@ -29,14 +29,15 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -47,20 +48,19 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
*******************************************************************************/
public class RecordLookupHelper
{
private final AbstractActionInput actionInput;
private Map<String, Map<Serializable, QRecord>> recordMaps = new HashMap<>();
private Set<String> preloadedKeys = new HashSet<>();
private Set<Pair<String, String>> disallowedOneOffLookups = new HashSet<>();
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public RecordLookupHelper(AbstractActionInput actionInput)
public RecordLookupHelper()
{
this.actionInput = actionInput;
}
@ -76,13 +76,16 @@ public class RecordLookupHelper
////////////////////////////////////////////////////////////
// make sure we have they key object in the expected type //
////////////////////////////////////////////////////////////
QFieldType type = actionInput.getInstance().getTable(tableName).getField(keyFieldName).getType();
QFieldType type = QContext.getQInstance().getTable(tableName).getField(keyFieldName).getType();
key = ValueUtils.getValueAsFieldType(type, key);
if(!recordMap.containsKey(key))
{
Optional<QRecord> optRecord = GeneralProcessUtils.getRecordByField(actionInput, tableName, keyFieldName, key);
recordMap.put(key, optRecord.orElse(null));
if(disallowedOneOffLookups.isEmpty() || !disallowedOneOffLookups.contains(Pair.of(tableName, keyFieldName)))
{
Optional<QRecord> optRecord = GeneralProcessUtils.getRecordByField(null, tableName, keyFieldName, key);
recordMap.put(key, optRecord.orElse(null));
}
}
return (recordMap.get(key));
@ -103,7 +106,7 @@ public class RecordLookupHelper
String mapKey = tableName + "." + keyFieldName;
if(!preloadedKeys.contains(mapKey))
{
Map<Serializable, QRecord> recordMap = GeneralProcessUtils.loadTableToMap(actionInput, tableName, keyFieldName);
Map<Serializable, QRecord> recordMap = GeneralProcessUtils.loadTableToMap(null, tableName, keyFieldName);
recordMaps.put(mapKey, recordMap);
preloadedKeys.add(mapKey);
}
@ -123,7 +126,7 @@ public class RecordLookupHelper
{
String mapKey = tableName + "." + keyFieldName;
Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>());
tableMap.putAll(GeneralProcessUtils.loadTableToMap(actionInput, tableName, keyFieldName, filter));
tableMap.putAll(GeneralProcessUtils.loadTableToMap(null, tableName, keyFieldName, filter));
}
@ -134,20 +137,20 @@ public class RecordLookupHelper
** aren't found, then a null will be cached (for each element in the inList).
**
*******************************************************************************/
public void preloadRecords(String tableName, String keyFieldName, List<Serializable> inList) throws QException
public Map<Serializable, QRecord> preloadRecords(String tableName, String keyFieldName, List<Serializable> inList) throws QException
{
if(CollectionUtils.nullSafeIsEmpty(inList))
{
return;
return null;
}
String mapKey = tableName + "." + keyFieldName;
Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>());
QQueryFilter filter = new QQueryFilter(new QFilterCriteria(keyFieldName, QCriteriaOperator.IN, inList));
tableMap.putAll(GeneralProcessUtils.loadTableToMap(actionInput, tableName, keyFieldName, filter));
tableMap.putAll(GeneralProcessUtils.loadTableToMap(null, tableName, keyFieldName, filter));
QFieldType type = actionInput.getInstance().getTable(tableName).getField(keyFieldName).getType();
QFieldType type = QContext.getQInstance().getTable(tableName).getField(keyFieldName).getType();
for(Serializable keyValue : inList)
{
if(!tableMap.containsKey(keyValue))
@ -156,6 +159,8 @@ public class RecordLookupHelper
tableMap.put(keyValue, null);
}
}
return (tableMap);
}
@ -198,7 +203,7 @@ public class RecordLookupHelper
*******************************************************************************/
public Serializable getRecordId(String tableName, String keyFieldName, Serializable key) throws QException
{
String primaryKeyField = actionInput.getInstance().getTable(tableName).getPrimaryKeyField();
String primaryKeyField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField();
return (getRecordValue(tableName, primaryKeyField, keyFieldName, key));
}
@ -215,4 +220,14 @@ public class RecordLookupHelper
return (ValueUtils.getValueAsType(type, value));
}
/*******************************************************************************
**
*******************************************************************************/
public void setMayNotDoOneOffLookups(String tableName, String fieldName)
{
this.disallowedOneOffLookups.add(Pair.of(tableName, fieldName));
}
}

56
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java
View File

@ -26,7 +26,6 @@ import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
@ -65,7 +64,7 @@ class RecordLookupHelperTest extends BaseTest
{
QInstance qInstance = QContext.getQInstance();
TestUtils.insertDefaultShapes(qInstance);
RecordLookupHelper recordLookupHelper = new RecordLookupHelper(new AbstractActionInput());
RecordLookupHelper recordLookupHelper = new RecordLookupHelper();
MemoryRecordStore.setCollectStatistics(true);
assertEquals(2, recordLookupHelper.getRecordId(TestUtils.TABLE_NAME_SHAPE, "name", "Square"));
@ -92,7 +91,7 @@ class RecordLookupHelperTest extends BaseTest
QInstance qInstance = QContext.getQInstance();
TestUtils.insertDefaultShapes(qInstance);
RecordLookupHelper recordLookupHelper = new RecordLookupHelper(new AbstractActionInput());
RecordLookupHelper recordLookupHelper = new RecordLookupHelper();
recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name");
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
@ -133,8 +132,8 @@ class RecordLookupHelperTest extends BaseTest
QInstance qInstance = QContext.getQInstance();
TestUtils.insertDefaultShapes(qInstance);
RecordLookupHelper recordLookupHelper = new RecordLookupHelper(new AbstractActionInput());
recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name", List.of("Triangle", "Square", "Circle", "Hexagon"));
RecordLookupHelper recordLookupHelper = new RecordLookupHelper();
recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name", List.of("Square", "Circle"));
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
//////////////////////////////////////////////
@ -149,9 +148,50 @@ class RecordLookupHelperTest extends BaseTest
//////////////////////////////////////////////////
assertNull(recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Hexagon"));
/////////////////////////////////////////////////////
// all those gets should run no additional queries //
/////////////////////////////////////////////////////
////////////////////////////////////////////////
// those gets should run 2 additional queries //
////////////////////////////////////////////////
assertEquals(3, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWithPreloadAndDisabledOneOffLookups() throws QException
{
QInstance qInstance = QContext.getQInstance();
TestUtils.insertDefaultShapes(qInstance);
RecordLookupHelper recordLookupHelper = new RecordLookupHelper();
recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name", List.of("Triangle", "Square", "Octagon"));
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
////////////////////////////////////////////////////////
// this is the key thing being tested in this method. //
////////////////////////////////////////////////////////
recordLookupHelper.setMayNotDoOneOffLookups(TestUtils.TABLE_NAME_SHAPE, "name");
////////////////////////////////////////////////////////////
// assert we do not find a record if it was not preloaded //
////////////////////////////////////////////////////////////
assertNull(recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Circle"));
//////////////////////////////////////////////////
// assert we cached a null for a name not found //
//////////////////////////////////////////////////
assertNull(recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Octagon"));
//////////////////////////////////////////////////////////////////////////////////////////
// assert we do not try to look up a name that we didn't rep-load, and that isn't found //
//////////////////////////////////////////////////////////////////////////////////////////
assertNull(recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Hexagon"));
//////////////////////////////////////////////////////////
// there shouldn't have been any additional queries ran //
//////////////////////////////////////////////////////////
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
}