diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipeBufferedWrapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipeBufferedWrapper.java new file mode 100644 index 00000000..08117c06 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipeBufferedWrapper.java @@ -0,0 +1,79 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** Subclass of BufferedRecordPipe, which ultimately sends records down to an + ** original RecordPipe. + ** + ** Meant to be used where: someone passed in a RecordPipe (so they have a reference + ** to it, and they are waiting to read from it), but the producer knows that + ** it will be better to buffer the records, so they want to use a buffered pipe + ** (but they still need the records to end up in the original pipe - thus - + ** it gets wrapped by an object of this class). + *******************************************************************************/ +public class RecordPipeBufferedWrapper extends BufferedRecordPipe +{ + private RecordPipe wrappedPipe; + + + + /******************************************************************************* + ** Constructor - uses default buffer size + ** + *******************************************************************************/ + public RecordPipeBufferedWrapper(RecordPipe wrappedPipe) + { + this.wrappedPipe = wrappedPipe; + } + + + + /******************************************************************************* + ** Constructor - customize buffer size. + ** + *******************************************************************************/ + public RecordPipeBufferedWrapper(Integer bufferSize, RecordPipe wrappedPipe) + { + super(bufferSize); + this.wrappedPipe = wrappedPipe; + } + + + + /******************************************************************************* + ** when it's time to actually add records into the pipe, actually add them + ** into the wrapped pipe! + *******************************************************************************/ + @Override + public void addRecords(List records) throws QException + { + wrappedPipe.addRecords(records); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java index a90535f9..1a5e5c84 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCusto import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.reporting.BufferedRecordPipe; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipeBufferedWrapper; import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.context.QContext; @@ -83,6 +84,15 @@ public class QueryAction if(queryInput.getRecordPipe() != null) { queryInput.getRecordPipe().setPostRecordActions(this::postRecordActions); + + if(queryInput.getIncludeAssociations()) + { + ////////////////////////////////////////////////////////////////////////////////////////// + // if the user requested to include associations, it's important that that is buffered, // + // (for performance reasons), so, wrap the user's pipe with a buffer // + ////////////////////////////////////////////////////////////////////////////////////////// + queryInput.setRecordPipe(new RecordPipeBufferedWrapper(queryInput.getRecordPipe())); + } } QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); @@ -111,6 +121,7 @@ public class QueryAction *******************************************************************************/ private void manageAssociations(QueryInput queryInput, List queryOutputRecords) throws QException { + LOG.info("In manageAssociations for " + queryInput.getTableName() + " with " + queryOutputRecords.size() + " records"); QTableMetaData table = queryInput.getTable(); for(Association association : CollectionUtils.nonNullList(table.getAssociations())) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java index c69a2fc0..ae44f8fe 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java @@ -261,7 +261,7 @@ public class ValidateRecordSecurityLockHelper QSecurityKeyType securityKeyType = QContext.getQInstance().getSecurityKeyType(recordSecurityLock.getSecurityKeyType()); if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()) && QContext.getQSession().hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN)) { - LOG.debug("Session has " + securityKeyType.getAllAccessKeyName() + " - not checking this lock."); + LOG.trace("Session has " + securityKeyType.getAllAccessKeyName() + " - not checking this lock."); } else { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryQueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryQueryAction.java index cd0e8bf7..962fe46a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryQueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryQueryAction.java @@ -26,6 +26,7 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; /******************************************************************************* @@ -43,7 +44,16 @@ public class MemoryQueryAction implements QueryInterface try { QueryOutput queryOutput = new QueryOutput(queryInput); - queryOutput.addRecords(MemoryRecordStore.getInstance().query(queryInput)); + + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // add the records to the output one-by-one -- this more closely matches how "real" backends perform // + // and works better w/ pipes // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QRecord qRecord : MemoryRecordStore.getInstance().query(queryInput)) + { + queryOutput.addRecord(qRecord); + } + return (queryOutput); } catch(Exception e) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java index 9c70a049..8051f33a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.actions.tables; import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop; import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -200,6 +201,41 @@ class QueryActionTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueryManyRecordsAssociationsWithPipe() throws QException + { + QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true); + insertNOrdersWithAssociations(2500); + + RecordPipe pipe = new RecordPipe(1000); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.setRecordPipe(pipe); + queryInput.setIncludeAssociations(true); + + int recordsConsumed = new AsyncRecordPipeLoop().run("Test", null, pipe, (callback) -> + { + new QueryAction().execute(queryInput); + return (true); + }, () -> + { + List records = pipe.consumeAvailableRecords(); + for(QRecord record : records) + { + assertEquals(1, record.getAssociatedRecords().get("orderLine").size()); + assertEquals(1, record.getAssociatedRecords().get("extrinsics").size()); + } + return (records.size()); + }); + + assertEquals(2500, recordsConsumed); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -356,4 +392,25 @@ class QueryActionTest extends BaseTest )); new InsertAction().execute(insertInput); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void insertNOrdersWithAssociations(int n) throws QException + { + List recordList = new ArrayList<>(); + for(int i = 0; i < n; i++) + { + recordList.add(new QRecord().withValue("storeId", 1).withValue("orderNo", "ORD" + i) + .withAssociatedRecord("orderLine", new QRecord().withValue("sku", "BASIC1").withValue("quantity", 3)) + .withAssociatedRecord("extrinsics", new QRecord().withValue("key", "YOUR-FIELD").withValue("value", "YOUR-VALUE"))); + } + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_ORDER); + insertInput.setRecords(recordList); + new InsertAction().execute(insertInput); + } }