QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-17 20:50:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Do pagination, to avoid queries with, idk, 320,000 params...

Browse Source
This commit is contained in:
darin.kelkhoff
2024-12-19 12:07:12 -06:00
parent 96761b7162
commit 11ff517769
2 changed files with 196 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java
View File

@ -50,6 +50,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
*******************************************************************************/
public class UniqueKeyHelper
{
private static Integer pageSize = 1000;
/*******************************************************************************
**
@ -60,62 +61,71 @@ public class UniqueKeyHelper
Map<List<Serializable>, Serializable> existingRecords = new HashMap<>();
if(ukFieldNames != null)
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(table.getName());
queryInput.setTransaction(transaction);
for(List<QRecord> page : CollectionUtils.getPages(recordList, pageSize))
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(table.getName());
queryInput.setTransaction(transaction);
QQueryFilter filter = new QQueryFilter();
if(ukFieldNames.size() == 1)
{
List<Serializable> values = recordList.stream()
.filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors()))
.map(r -> r.getValue(ukFieldNames.get(0)))
.collect(Collectors.toList());
filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values));
}
else
{
filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
for(QRecord record : recordList)
QQueryFilter filter = new QQueryFilter();
if(ukFieldNames.size() == 1)
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
List<Serializable> values = page.stream()
.filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors()))
.map(r -> r.getValue(ukFieldNames.get(0)))
.collect(Collectors.toList());
if(values.isEmpty())
{
continue;
}
QQueryFilter subFilter = new QQueryFilter();
filter.addSubFilter(subFilter);
for(String fieldName : ukFieldNames)
filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values));
}
else
{
filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
for(QRecord record : page)
{
Serializable value = record.getValue(fieldName);
if(value == null)
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK));
continue;
}
else
QQueryFilter subFilter = new QQueryFilter();
filter.addSubFilter(subFilter);
for(String fieldName : ukFieldNames)
{
subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value));
Serializable value = record.getValue(fieldName);
if(value == null)
{
subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK));
}
else
{
subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value));
}
}
}
if(CollectionUtils.nullSafeIsEmpty(filter.getSubFilters()))
{
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if we didn't build any sub-filters (because all records have errors in them), don't run a query w/ no clauses - continue to next page //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
continue;
}
}
if(CollectionUtils.nullSafeIsEmpty(filter.getSubFilters()))
queryInput.setFilter(filter);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
for(QRecord record : queryOutput.getRecords())
{
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if we didn't build any sub-filters (because all records have errors in them), don't run a query w/ no clauses - rather - return early. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
return (existingRecords);
}
}
queryInput.setFilter(filter);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
for(QRecord record : queryOutput.getRecords())
{
Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
if(keyValues.isPresent())
{
existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField()));
Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
if(keyValues.isPresent())
{
existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField()));
}
}
}
}
@ -200,4 +210,26 @@ public class UniqueKeyHelper
}
}
/*******************************************************************************
** Getter for pageSize
**
*******************************************************************************/
public static Integer getPageSize()
{
return pageSize;
}
/*******************************************************************************
** Setter for pageSize
**
*******************************************************************************/
public static void setPageSize(Integer pageSize)
{
UniqueKeyHelper.pageSize = pageSize;
}
}

123
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelperTest.java Normal file
View File

@ -0,0 +1,123 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.tables.helpers;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for UniqueKeyHelper
*******************************************************************************/
class UniqueKeyHelperTest extends BaseTest
{
private static Integer originalPageSize;
/*******************************************************************************
**
*******************************************************************************/
@BeforeAll
static void beforeAll()
{
originalPageSize = UniqueKeyHelper.getPageSize();
UniqueKeyHelper.setPageSize(5);
}
/*******************************************************************************
**
*******************************************************************************/
@AfterAll
static void afterAll()
{
UniqueKeyHelper.setPageSize(originalPageSize);
}
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.fullReset();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testUniqueKey() throws QException
{
List<QRecord> recordsWithKey1Equals1AndKey2In1Through10 = List.of(
new QRecord().withValue("key1", 1).withValue("key2", 1),
new QRecord().withValue("key1", 1).withValue("key2", 2),
new QRecord().withValue("key1", 1).withValue("key2", 3),
new QRecord().withValue("key1", 1).withValue("key2", 4),
new QRecord().withValue("key1", 1).withValue("key2", 5),
new QRecord().withValue("key1", 1).withValue("key2", 6),
new QRecord().withValue("key1", 1).withValue("key2", 7),
new QRecord().withValue("key1", 1).withValue("key2", 8),
new QRecord().withValue("key1", 1).withValue("key2", 9),
new QRecord().withValue("key1", 1).withValue("key2", 10)
);
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_TWO_KEYS);
insertInput.setRecords(recordsWithKey1Equals1AndKey2In1Through10);
InsertOutput insertOutput = new InsertAction().execute(insertInput);
MemoryRecordStore.resetStatistics();
MemoryRecordStore.setCollectStatistics(true);
QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_TWO_KEYS);
Map<List<Serializable>, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(null, table, recordsWithKey1Equals1AndKey2In1Through10, table.getUniqueKeys().get(0), false);
assertEquals(recordsWithKey1Equals1AndKey2In1Through10.size(), existingKeys.size());
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
}
}