Add ability to disable one-off lookups

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java

@@ -301,7 +301,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
     *******************************************************************************/
    protected void initializeRecordLookupHelper(RunBackendStepInput runBackendStepInput, List<QRecord> sourceRecordList) throws QException
    {
-      this.recordLookupHelper = new RecordLookupHelper(runBackendStepInput);
+      this.recordLookupHelper = new RecordLookupHelper();
 
       /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // if there's only 1 record, don't bother preloading all records - just do the single lookup by the single needed key. //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java

@@ -29,14 +29,15 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.Pair;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
@@ -47,20 +48,19 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
  *******************************************************************************/
 public class RecordLookupHelper
 {
-   private final AbstractActionInput actionInput;
-
    private Map<String, Map<Serializable, QRecord>> recordMaps    = new HashMap<>();
    private Set<String>                             preloadedKeys = new HashSet<>();
 
+   private Set<Pair<String, String>> disallowedOneOffLookups = new HashSet<>();
+
 
 
    /*******************************************************************************
     ** Constructor
     **
     *******************************************************************************/
-   public RecordLookupHelper(AbstractActionInput actionInput)
+   public RecordLookupHelper()
    {
-      this.actionInput = actionInput;
    }
 
 
@@ -76,13 +76,16 @@ public class RecordLookupHelper
       ////////////////////////////////////////////////////////////
       // make sure we have they key object in the expected type //
       ////////////////////////////////////////////////////////////
-      QFieldType type = actionInput.getInstance().getTable(tableName).getField(keyFieldName).getType();
+      QFieldType type = QContext.getQInstance().getTable(tableName).getField(keyFieldName).getType();
       key = ValueUtils.getValueAsFieldType(type, key);
 
       if(!recordMap.containsKey(key))
       {
-         Optional<QRecord> optRecord = GeneralProcessUtils.getRecordByField(actionInput, tableName, keyFieldName, key);
+         if(disallowedOneOffLookups.isEmpty() || !disallowedOneOffLookups.contains(Pair.of(tableName, keyFieldName)))
-         recordMap.put(key, optRecord.orElse(null));
+         {
+            Optional<QRecord> optRecord = GeneralProcessUtils.getRecordByField(null, tableName, keyFieldName, key);
+            recordMap.put(key, optRecord.orElse(null));
+         }
       }
 
       return (recordMap.get(key));
@@ -103,7 +106,7 @@ public class RecordLookupHelper
       String mapKey = tableName + "." + keyFieldName;
       if(!preloadedKeys.contains(mapKey))
       {
-         Map<Serializable, QRecord> recordMap = GeneralProcessUtils.loadTableToMap(actionInput, tableName, keyFieldName);
+         Map<Serializable, QRecord> recordMap = GeneralProcessUtils.loadTableToMap(null, tableName, keyFieldName);
          recordMaps.put(mapKey, recordMap);
          preloadedKeys.add(mapKey);
       }
@@ -123,7 +126,7 @@ public class RecordLookupHelper
    {
       String                     mapKey   = tableName + "." + keyFieldName;
       Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>());
-      tableMap.putAll(GeneralProcessUtils.loadTableToMap(actionInput, tableName, keyFieldName, filter));
+      tableMap.putAll(GeneralProcessUtils.loadTableToMap(null, tableName, keyFieldName, filter));
    }
 
 
@@ -134,20 +137,20 @@ public class RecordLookupHelper
     ** aren't found, then a null will be cached (for each element in the inList).
     **
     *******************************************************************************/
-   public void preloadRecords(String tableName, String keyFieldName, List<Serializable> inList) throws QException
+   public Map<Serializable, QRecord> preloadRecords(String tableName, String keyFieldName, List<Serializable> inList) throws QException
    {
       if(CollectionUtils.nullSafeIsEmpty(inList))
       {
-         return;
+         return null;
       }
 
       String                     mapKey   = tableName + "." + keyFieldName;
       Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>());
 
       QQueryFilter filter = new QQueryFilter(new QFilterCriteria(keyFieldName, QCriteriaOperator.IN, inList));
-      tableMap.putAll(GeneralProcessUtils.loadTableToMap(actionInput, tableName, keyFieldName, filter));
+      tableMap.putAll(GeneralProcessUtils.loadTableToMap(null, tableName, keyFieldName, filter));
 
-      QFieldType type = actionInput.getInstance().getTable(tableName).getField(keyFieldName).getType();
+      QFieldType type = QContext.getQInstance().getTable(tableName).getField(keyFieldName).getType();
       for(Serializable keyValue : inList)
       {
          if(!tableMap.containsKey(keyValue))
@@ -156,6 +159,8 @@ public class RecordLookupHelper
             tableMap.put(keyValue, null);
          }
       }
+
+      return (tableMap);
    }
 
 
@@ -198,7 +203,7 @@ public class RecordLookupHelper
     *******************************************************************************/
    public Serializable getRecordId(String tableName, String keyFieldName, Serializable key) throws QException
    {
-      String primaryKeyField = actionInput.getInstance().getTable(tableName).getPrimaryKeyField();
+      String primaryKeyField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField();
       return (getRecordValue(tableName, primaryKeyField, keyFieldName, key));
    }
 
@@ -215,4 +220,14 @@ public class RecordLookupHelper
       return (ValueUtils.getValueAsType(type, value));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setMayNotDoOneOffLookups(String tableName, String fieldName)
+   {
+      this.disallowedOneOffLookups.add(Pair.of(tableName, fieldName));
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java

@@ -26,7 +26,6 @@ import java.util.List;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
@@ -65,7 +64,7 @@ class RecordLookupHelperTest extends BaseTest
    {
       QInstance qInstance = QContext.getQInstance();
       TestUtils.insertDefaultShapes(qInstance);
-      RecordLookupHelper recordLookupHelper = new RecordLookupHelper(new AbstractActionInput());
+      RecordLookupHelper recordLookupHelper = new RecordLookupHelper();
 
       MemoryRecordStore.setCollectStatistics(true);
       assertEquals(2, recordLookupHelper.getRecordId(TestUtils.TABLE_NAME_SHAPE, "name", "Square"));
@@ -92,7 +91,7 @@ class RecordLookupHelperTest extends BaseTest
       QInstance qInstance = QContext.getQInstance();
       TestUtils.insertDefaultShapes(qInstance);
 
-      RecordLookupHelper recordLookupHelper = new RecordLookupHelper(new AbstractActionInput());
+      RecordLookupHelper recordLookupHelper = new RecordLookupHelper();
       recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name");
       assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
 
@@ -133,8 +132,8 @@ class RecordLookupHelperTest extends BaseTest
       QInstance qInstance = QContext.getQInstance();
       TestUtils.insertDefaultShapes(qInstance);
 
-      RecordLookupHelper recordLookupHelper = new RecordLookupHelper(new AbstractActionInput());
+      RecordLookupHelper recordLookupHelper = new RecordLookupHelper();
-      recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name", List.of("Triangle", "Square", "Circle", "Hexagon"));
+      recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name", List.of("Square", "Circle"));
       assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
 
       //////////////////////////////////////////////
@@ -149,9 +148,50 @@ class RecordLookupHelperTest extends BaseTest
       //////////////////////////////////////////////////
       assertNull(recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Hexagon"));
 
-      /////////////////////////////////////////////////////
+      ////////////////////////////////////////////////
-      // all those gets should run no additional queries //
+      // those gets should run 2 additional queries //
-      /////////////////////////////////////////////////////
+      ////////////////////////////////////////////////
+      assertEquals(3, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testWithPreloadAndDisabledOneOffLookups() throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+      TestUtils.insertDefaultShapes(qInstance);
+
+      RecordLookupHelper recordLookupHelper = new RecordLookupHelper();
+      recordLookupHelper.preloadRecords(TestUtils.TABLE_NAME_SHAPE, "name", List.of("Triangle", "Square", "Octagon"));
+      assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
+
+      ////////////////////////////////////////////////////////
+      // this is the key thing being tested in this method. //
+      ////////////////////////////////////////////////////////
+      recordLookupHelper.setMayNotDoOneOffLookups(TestUtils.TABLE_NAME_SHAPE, "name");
+
+      ////////////////////////////////////////////////////////////
+      // assert we do not find a record if it was not preloaded //
+      ////////////////////////////////////////////////////////////
+      assertNull(recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Circle"));
+
+      //////////////////////////////////////////////////
+      // assert we cached a null for a name not found //
+      //////////////////////////////////////////////////
+      assertNull(recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Octagon"));
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // assert we do not try to look up a name that we didn't rep-load, and that isn't found //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      assertNull(recordLookupHelper.getRecordByKey(TestUtils.TABLE_NAME_SHAPE, "name", "Hexagon"));
+
+      //////////////////////////////////////////////////////////
+      // there shouldn't have been any additional queries ran //
+      //////////////////////////////////////////////////////////
       assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
    }