From b0cca3f1d7d547b3668ed053b32f67c5cb014c02 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 8 Feb 2023 17:02:06 -0600 Subject: [PATCH] Add ability to disable one-off lookups --- .../AbstractTableSyncTransformStep.java | 2 +- .../processes/utils/RecordLookupHelper.java | 45 ++++++++++----- .../utils/RecordLookupHelperTest.java | 56 ++++++++++++++++--- 3 files changed, 79 insertions(+), 24 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java index 62ea1d33..cd8c45e4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -301,7 +301,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt *******************************************************************************/ protected void initializeRecordLookupHelper(RunBackendStepInput runBackendStepInput, List sourceRecordList) throws QException { - this.recordLookupHelper = new RecordLookupHelper(runBackendStepInput); + this.recordLookupHelper = new RecordLookupHelper(); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if there's only 1 record, don't bother preloading all records - just do the single lookup by the single needed key. // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java index 5b7ddaa9..02f4e84f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java @@ -29,14 +29,15 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -47,20 +48,19 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public class RecordLookupHelper { - private final AbstractActionInput actionInput; - private Map> recordMaps = new HashMap<>(); private Set preloadedKeys = new HashSet<>(); + private Set> disallowedOneOffLookups = new HashSet<>(); + /******************************************************************************* ** Constructor ** *******************************************************************************/ - public RecordLookupHelper(AbstractActionInput actionInput) + public RecordLookupHelper() { - this.actionInput = actionInput; } @@ -76,13 +76,16 @@ public class RecordLookupHelper //////////////////////////////////////////////////////////// // make sure we have they key object in the expected type // //////////////////////////////////////////////////////////// - QFieldType type = actionInput.getInstance().getTable(tableName).getField(keyFieldName).getType(); + QFieldType type = QContext.getQInstance().getTable(tableName).getField(keyFieldName).getType(); key = ValueUtils.getValueAsFieldType(type, key); if(!recordMap.containsKey(key)) { - Optional optRecord = GeneralProcessUtils.getRecordByField(actionInput, tableName, keyFieldName, key); - recordMap.put(key, optRecord.orElse(null)); + if(disallowedOneOffLookups.isEmpty() || !disallowedOneOffLookups.contains(Pair.of(tableName, keyFieldName))) + { + Optional optRecord = GeneralProcessUtils.getRecordByField(null, tableName, keyFieldName, key); + recordMap.put(key, optRecord.orElse(null)); + } } return (recordMap.get(key)); @@ -103,7 +106,7 @@ public class RecordLookupHelper String mapKey = tableName + "." + keyFieldName; if(!preloadedKeys.contains(mapKey)) { - Map recordMap = GeneralProcessUtils.loadTableToMap(actionInput, tableName, keyFieldName); + Map recordMap = GeneralProcessUtils.loadTableToMap(null, tableName, keyFieldName); recordMaps.put(mapKey, recordMap); preloadedKeys.add(mapKey); } @@ -123,7 +126,7 @@ public class RecordLookupHelper { String mapKey = tableName + "." + keyFieldName; Map tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>()); - tableMap.putAll(GeneralProcessUtils.loadTableToMap(actionInput, tableName, keyFieldName, filter)); + tableMap.putAll(GeneralProcessUtils.loadTableToMap(null, tableName, keyFieldName, filter)); } @@ -134,20 +137,20 @@ public class RecordLookupHelper ** aren't found, then a null will be cached (for each element in the inList). ** *******************************************************************************/ - public void preloadRecords(String tableName, String keyFieldName, List inList) throws QException + public Map preloadRecords(String tableName, String keyFieldName, List inList) throws QException { if(CollectionUtils.nullSafeIsEmpty(inList)) { - return; + return null; } String mapKey = tableName + "." + keyFieldName; Map tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>()); QQueryFilter filter = new QQueryFilter(new QFilterCriteria(keyFieldName, QCriteriaOperator.IN, inList)); - tableMap.putAll(GeneralProcessUtils.loadTableToMap(actionInput, tableName, keyFieldName, filter)); + tableMap.putAll(GeneralProcessUtils.loadTableToMap(null, tableName, keyFieldName, filter)); - QFieldType type = actionInput.getInstance().getTable(tableName).getField(keyFieldName).getType(); + QFieldType type = QContext.getQInstance().getTable(tableName).getField(keyFieldName).getType(); for(Serializable keyValue : inList) { if(!tableMap.containsKey(keyValue)) @@ -156,6 +159,8 @@ public class RecordLookupHelper tableMap.put(keyValue, null); } } + + return (tableMap); } @@ -198,7 +203,7 @@ public class RecordLookupHelper *******************************************************************************/ public Serializable getRecordId(String tableName, String keyFieldName, Serializable key) throws QException { - String primaryKeyField = actionInput.getInstance().getTable(tableName).getPrimaryKeyField(); + String primaryKeyField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField(); return (getRecordValue(tableName, primaryKeyField, keyFieldName, key)); } @@ -215,4 +220,14 @@ public class RecordLookupHelper return (ValueUtils.getValueAsType(type, value)); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public void setMayNotDoOneOffLookups(String tableName, String fieldName) + { + this.disallowedOneOffLookups.add(Pair.of(tableName, fieldName)); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java index 73e7a666..4e2f3b30 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java @@ -26,7 +26,6 @@ import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qqq.backend.core.utils.TestUtils; @@ -65,7 +64,7 @@ class RecordLookupHelperTest extends BaseTest { QInstance qInstance = QContext.getQInstance(); TestUtils.insertDefaultShapes(qInstance); - RecordLookupHelper recordLookupHelper = new RecordLookupHelper(new AbstractActionInput()); + RecordLookupHelper recordLookupHelper = new RecordLookupHelper(); MemoryRecordStore.setCollectStatistics(true); assertEquals(2, recordLookupHelper.getRecordId(TestUtils.TABLE_NAME_SHAPE, "name", "Square")); @@ -92,7 +91,7 @@ class RecordLookupHelperTest extends BaseTest QInstance qInstance = QContext.getQInstance(); TestUtils.insertDefaultShapes(qInstance); - RecordLookupHelper recordLookupHelper = new RecordLookupHelper(new AbstractActionInput()); + RecordLookupHelper recordLookupHelper = new RecordLookupHelper(); recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name"); assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN)); @@ -133,8 +132,8 @@ class RecordLookupHelperTest extends BaseTest QInstance qInstance = QContext.getQInstance(); TestUtils.insertDefaultShapes(qInstance); - RecordLookupHelper recordLookupHelper = new RecordLookupHelper(new AbstractActionInput()); - recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name", List.of("Triangle", "Square", "Circle", "Hexagon")); + RecordLookupHelper recordLookupHelper = new RecordLookupHelper(); + recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name", List.of("Square", "Circle")); assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN)); ////////////////////////////////////////////// @@ -149,9 +148,50 @@ class RecordLookupHelperTest extends BaseTest ////////////////////////////////////////////////// assertNull(recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Hexagon")); - ///////////////////////////////////////////////////// - // all those gets should run no additional queries // - ///////////////////////////////////////////////////// + //////////////////////////////////////////////// + // those gets should run 2 additional queries // + //////////////////////////////////////////////// + assertEquals(3, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWithPreloadAndDisabledOneOffLookups() throws QException + { + QInstance qInstance = QContext.getQInstance(); + TestUtils.insertDefaultShapes(qInstance); + + RecordLookupHelper recordLookupHelper = new RecordLookupHelper(); + recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name", List.of("Triangle", "Square", "Octagon")); + assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN)); + + //////////////////////////////////////////////////////// + // this is the key thing being tested in this method. // + //////////////////////////////////////////////////////// + recordLookupHelper.setMayNotDoOneOffLookups(TestUtils.TABLE_NAME_SHAPE, "name"); + + //////////////////////////////////////////////////////////// + // assert we do not find a record if it was not preloaded // + //////////////////////////////////////////////////////////// + assertNull(recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Circle")); + + ////////////////////////////////////////////////// + // assert we cached a null for a name not found // + ////////////////////////////////////////////////// + assertNull(recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Octagon")); + + ////////////////////////////////////////////////////////////////////////////////////////// + // assert we do not try to look up a name that we didn't rep-load, and that isn't found // + ////////////////////////////////////////////////////////////////////////////////////////// + assertNull(recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Hexagon")); + + ////////////////////////////////////////////////////////// + // there shouldn't have been any additional queries ran // + ////////////////////////////////////////////////////////// assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN)); }