Do pagination, to avoid queries with, idk, 320,000 params...

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java

@@ -50,6 +50,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
  *******************************************************************************/
 public class UniqueKeyHelper
 {
+   private static Integer pageSize = 1000;
 
    /*******************************************************************************
     **
@@ -60,62 +61,71 @@ public class UniqueKeyHelper
       Map<List<Serializable>, Serializable> existingRecords = new HashMap<>();
       if(ukFieldNames != null)
       {
-         QueryInput queryInput = new QueryInput();
+         for(List<QRecord> page : CollectionUtils.getPages(recordList, pageSize))
-         queryInput.setTableName(table.getName());
+         {
-         queryInput.setTransaction(transaction);
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(table.getName());
+            queryInput.setTransaction(transaction);
 
-         QQueryFilter filter = new QQueryFilter();
+            QQueryFilter filter = new QQueryFilter();
-         if(ukFieldNames.size() == 1)
+            if(ukFieldNames.size() == 1)
-         {
-            List<Serializable> values = recordList.stream()
-               .filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors()))
-               .map(r -> r.getValue(ukFieldNames.get(0)))
-               .collect(Collectors.toList());
-            filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values));
-         }
-         else
-         {
-            filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
-            for(QRecord record : recordList)
             {
-               if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+               List<Serializable> values = page.stream()
+                  .filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors()))
+                  .map(r -> r.getValue(ukFieldNames.get(0)))
+                  .collect(Collectors.toList());
+
+               if(values.isEmpty())
                {
                   continue;
                }
 
-               QQueryFilter subFilter = new QQueryFilter();
+               filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values));
-               filter.addSubFilter(subFilter);
+            }
-               for(String fieldName : ukFieldNames)
+            else
+            {
+               filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
+               for(QRecord record : page)
                {
-                  Serializable value = record.getValue(fieldName);
+                  if(CollectionUtils.nullSafeHasContents(record.getErrors()))
-                  if(value == null)
                   {
-                     subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK));
+                     continue;
                   }
-                  else
+
+                  QQueryFilter subFilter = new QQueryFilter();
+                  filter.addSubFilter(subFilter);
+                  for(String fieldName : ukFieldNames)
                   {
-                     subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value));
+                     Serializable value = record.getValue(fieldName);
+                     if(value == null)
+                     {
+                        subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK));
+                     }
+                     else
+                     {
+                        subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value));
+                     }
                   }
                }
+
+               if(CollectionUtils.nullSafeIsEmpty(filter.getSubFilters()))
+               {
+                  ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // if we didn't build any sub-filters (because all records have errors in them), don't run a query w/ no clauses - continue to next page //
+                  ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  continue;
+               }
             }
 
-            if(CollectionUtils.nullSafeIsEmpty(filter.getSubFilters()))
+            queryInput.setFilter(filter);
+            QueryOutput queryOutput = new QueryAction().execute(queryInput);
+            for(QRecord record : queryOutput.getRecords())
             {
-               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
-               // if we didn't build any sub-filters (because all records have errors in them), don't run a query w/ no clauses - rather - return early. //
+               if(keyValues.isPresent())
-               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               {
-               return (existingRecords);
+                  existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField()));
-            }
+               }
-         }
-
-         queryInput.setFilter(filter);
-         QueryOutput queryOutput = new QueryAction().execute(queryInput);
-         for(QRecord record : queryOutput.getRecords())
-         {
-            Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
-            if(keyValues.isPresent())
-            {
-               existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField()));
             }
          }
       }
@@ -200,4 +210,26 @@ public class UniqueKeyHelper
       }
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for pageSize
+    **
+    *******************************************************************************/
+   public static Integer getPageSize()
+   {
+      return pageSize;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for pageSize
+    **
+    *******************************************************************************/
+   public static void setPageSize(Integer pageSize)
+   {
+      UniqueKeyHelper.pageSize = pageSize;
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelperTest.java

@@ -0,0 +1,123 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables.helpers;
+
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for UniqueKeyHelper 
+ *******************************************************************************/
+class UniqueKeyHelperTest extends BaseTest
+{
+   private static Integer originalPageSize;
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeAll
+   static void beforeAll()
+   {
+      originalPageSize = UniqueKeyHelper.getPageSize();
+      UniqueKeyHelper.setPageSize(5);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @AfterAll
+   static void afterAll()
+   {
+      UniqueKeyHelper.setPageSize(originalPageSize);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   @AfterEach
+   void beforeAndAfterEach()
+   {
+      MemoryRecordStore.fullReset();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testUniqueKey() throws QException
+   {
+      List<QRecord> recordsWithKey1Equals1AndKey2In1Through10 = List.of(
+         new QRecord().withValue("key1", 1).withValue("key2", 1),
+         new QRecord().withValue("key1", 1).withValue("key2", 2),
+         new QRecord().withValue("key1", 1).withValue("key2", 3),
+         new QRecord().withValue("key1", 1).withValue("key2", 4),
+         new QRecord().withValue("key1", 1).withValue("key2", 5),
+         new QRecord().withValue("key1", 1).withValue("key2", 6),
+         new QRecord().withValue("key1", 1).withValue("key2", 7),
+         new QRecord().withValue("key1", 1).withValue("key2", 8),
+         new QRecord().withValue("key1", 1).withValue("key2", 9),
+         new QRecord().withValue("key1", 1).withValue("key2", 10)
+      );
+
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_TWO_KEYS);
+      insertInput.setRecords(recordsWithKey1Equals1AndKey2In1Through10);
+      InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+      MemoryRecordStore.resetStatistics();
+      MemoryRecordStore.setCollectStatistics(true);
+
+      QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_TWO_KEYS);
+      Map<List<Serializable>, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(null, table, recordsWithKey1Equals1AndKey2In1Through10, table.getUniqueKeys().get(0), false);
+      assertEquals(recordsWithKey1Equals1AndKey2In1Through10.size(), existingKeys.size());
+
+      assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
+   }
+
+
+}