From 11ff51776951b340d5b37d56592b4fdfd757f4db Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 19 Dec 2024 12:07:12 -0600 Subject: [PATCH] Do pagination, to avoid queries with, idk, 320,000 params... --- .../tables/helpers/UniqueKeyHelper.java | 114 ++++++++++------ .../tables/helpers/UniqueKeyHelperTest.java | 123 ++++++++++++++++++ 2 files changed, 196 insertions(+), 41 deletions(-) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelperTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java index 7832344e..2e421a60 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java @@ -50,6 +50,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public class UniqueKeyHelper { + private static Integer pageSize = 1000; /******************************************************************************* ** @@ -60,62 +61,71 @@ public class UniqueKeyHelper Map, Serializable> existingRecords = new HashMap<>(); if(ukFieldNames != null) { - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(table.getName()); - queryInput.setTransaction(transaction); + for(List page : CollectionUtils.getPages(recordList, pageSize)) + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(table.getName()); + queryInput.setTransaction(transaction); - QQueryFilter filter = new QQueryFilter(); - if(ukFieldNames.size() == 1) - { - List values = recordList.stream() - .filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors())) - .map(r -> r.getValue(ukFieldNames.get(0))) - .collect(Collectors.toList()); - filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values)); - } - else - { - filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR); - for(QRecord record : recordList) + QQueryFilter filter = new QQueryFilter(); + if(ukFieldNames.size() == 1) { - if(CollectionUtils.nullSafeHasContents(record.getErrors())) + List values = page.stream() + .filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors())) + .map(r -> r.getValue(ukFieldNames.get(0))) + .collect(Collectors.toList()); + + if(values.isEmpty()) { continue; } - QQueryFilter subFilter = new QQueryFilter(); - filter.addSubFilter(subFilter); - for(String fieldName : ukFieldNames) + filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values)); + } + else + { + filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR); + for(QRecord record : page) { - Serializable value = record.getValue(fieldName); - if(value == null) + if(CollectionUtils.nullSafeHasContents(record.getErrors())) { - subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK)); + continue; } - else + + QQueryFilter subFilter = new QQueryFilter(); + filter.addSubFilter(subFilter); + for(String fieldName : ukFieldNames) { - subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value)); + Serializable value = record.getValue(fieldName); + if(value == null) + { + subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK)); + } + else + { + subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value)); + } } } + + if(CollectionUtils.nullSafeIsEmpty(filter.getSubFilters())) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we didn't build any sub-filters (because all records have errors in them), don't run a query w/ no clauses - continue to next page // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + continue; + } } - if(CollectionUtils.nullSafeIsEmpty(filter.getSubFilters())) + queryInput.setFilter(filter); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + for(QRecord record : queryOutput.getRecords()) { - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // if we didn't build any sub-filters (because all records have errors in them), don't run a query w/ no clauses - rather - return early. // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - return (existingRecords); - } - } - - queryInput.setFilter(filter); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - for(QRecord record : queryOutput.getRecords()) - { - Optional> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual); - if(keyValues.isPresent()) - { - existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField())); + Optional> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual); + if(keyValues.isPresent()) + { + existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField())); + } } } } @@ -200,4 +210,26 @@ public class UniqueKeyHelper } } + + + /******************************************************************************* + ** Getter for pageSize + ** + *******************************************************************************/ + public static Integer getPageSize() + { + return pageSize; + } + + + + /******************************************************************************* + ** Setter for pageSize + ** + *******************************************************************************/ + public static void setPageSize(Integer pageSize) + { + UniqueKeyHelper.pageSize = pageSize; + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelperTest.java new file mode 100644 index 00000000..b2a495c5 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelperTest.java @@ -0,0 +1,123 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.tables.helpers; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for UniqueKeyHelper + *******************************************************************************/ +class UniqueKeyHelperTest extends BaseTest +{ + private static Integer originalPageSize; + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeAll + static void beforeAll() + { + originalPageSize = UniqueKeyHelper.getPageSize(); + UniqueKeyHelper.setPageSize(5); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterAll + static void afterAll() + { + UniqueKeyHelper.setPageSize(originalPageSize); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.fullReset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testUniqueKey() throws QException + { + List recordsWithKey1Equals1AndKey2In1Through10 = List.of( + new QRecord().withValue("key1", 1).withValue("key2", 1), + new QRecord().withValue("key1", 1).withValue("key2", 2), + new QRecord().withValue("key1", 1).withValue("key2", 3), + new QRecord().withValue("key1", 1).withValue("key2", 4), + new QRecord().withValue("key1", 1).withValue("key2", 5), + new QRecord().withValue("key1", 1).withValue("key2", 6), + new QRecord().withValue("key1", 1).withValue("key2", 7), + new QRecord().withValue("key1", 1).withValue("key2", 8), + new QRecord().withValue("key1", 1).withValue("key2", 9), + new QRecord().withValue("key1", 1).withValue("key2", 10) + ); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_TWO_KEYS); + insertInput.setRecords(recordsWithKey1Equals1AndKey2In1Through10); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + + MemoryRecordStore.resetStatistics(); + MemoryRecordStore.setCollectStatistics(true); + + QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_TWO_KEYS); + Map, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(null, table, recordsWithKey1Equals1AndKey2In1Through10, table.getUniqueKeys().get(0), false); + assertEquals(recordsWithKey1Equals1AndKey2In1Through10.size(), existingKeys.size()); + + assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN)); + } + + +} \ No newline at end of file