Update to allow includeAssociations when querying into a record pipe. This meant propagating a lot of exceptions...

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/BufferedRecordPipe.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
 
 import java.util.ArrayList;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
@@ -63,7 +64,7 @@ public class BufferedRecordPipe extends RecordPipe
     **
     *******************************************************************************/
    @Override
-   public void addRecord(QRecord record)
+   public void addRecord(QRecord record) throws QException
    {
       buffer.add(record);
       if(buffer.size() >= bufferSize)
@@ -78,7 +79,7 @@ public class BufferedRecordPipe extends RecordPipe
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void finalFlush()
+   public void finalFlush() throws QException
    {
       if(!buffer.isEmpty())
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java

@@ -26,10 +26,11 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeConsumer;
 
 
 /*******************************************************************************
@@ -47,7 +48,7 @@ public class RecordPipe
 
    private boolean isTerminated = false;
 
-   private Consumer<List<QRecord>> postRecordActions = null;
+   private UnsafeConsumer<List<QRecord>, QException> postRecordActions = null;
 
    /////////////////////////////////////
    // See usage below for explanation //
@@ -93,7 +94,7 @@ public class RecordPipe
    /*******************************************************************************
     ** Add a record to the pipe.  Will block if the pipe is full.  Will noop if pipe is terminated.
     *******************************************************************************/
-   public void addRecord(QRecord record)
+   public void addRecord(QRecord record) throws QException
    {
       if(isTerminated)
       {
@@ -109,7 +110,7 @@ public class RecordPipe
          // (which we'll create as a field in this class, to avoid always re-constructing) //
          ////////////////////////////////////////////////////////////////////////////////////
          singleRecordListForPostRecordActions.add(record);
-         postRecordActions.accept(singleRecordListForPostRecordActions);
+         postRecordActions.run(singleRecordListForPostRecordActions);
          record = singleRecordListForPostRecordActions.remove(0);
       }
 
@@ -152,11 +153,11 @@ public class RecordPipe
    /*******************************************************************************
     ** Add a list of records to the pipe.  Will block if the pipe is full.  Will noop if pipe is terminated.
     *******************************************************************************/
-   public void addRecords(List<QRecord> records)
+   public void addRecords(List<QRecord> records) throws QException
    {
       if(postRecordActions != null)
       {
-         postRecordActions.accept(records);
+         postRecordActions.run(records);
       }
 
       //////////////////////////////////////////////////////////////////////////////////////////////////
@@ -207,7 +208,7 @@ public class RecordPipe
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void setPostRecordActions(Consumer<List<QRecord>> postRecordActions)
+   public void setPostRecordActions(UnsafeConsumer<List<QRecord>, QException> postRecordActions)
    {
       this.postRecordActions = postRecordActions;
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java

@@ -85,14 +85,6 @@ public class QueryAction
          queryInput.getRecordPipe().setPostRecordActions(this::postRecordActions);
       }
 
-      if(queryInput.getIncludeAssociations() && queryInput.getRecordPipe() != null)
-      {
-         //////////////////////////////////////////////
-         // todo - support this in the future maybe? //
-         //////////////////////////////////////////////
-         throw (new QException("Associations may not be fetched into a RecordPipe."));
-      }
-
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
       QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(queryInput.getBackend());
       // todo pre-customization - just get to modify the request?
@@ -109,11 +101,6 @@ public class QueryAction
          postRecordActions(queryOutput.getRecords());
       }
 
-      if(queryInput.getIncludeAssociations())
-      {
-         manageAssociations(queryInput, queryOutput);
-      }
-
       return queryOutput;
    }
 
@@ -122,7 +109,7 @@ public class QueryAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void manageAssociations(QueryInput queryInput, QueryOutput queryOutput) throws QException
+   private void manageAssociations(QueryInput queryInput, List<QRecord> queryOutputRecords) throws QException
    {
       QTableMetaData table = queryInput.getTable();
       for(Association association : CollectionUtils.nonNullList(table.getAssociations()))
@@ -147,7 +134,7 @@ public class QueryAction
             {
                JoinOn            joinOn = join.getJoinOns().get(0);
                Set<Serializable> values = new HashSet<>();
-               for(QRecord record : queryOutput.getRecords())
+               for(QRecord record : queryOutputRecords)
                {
                   Serializable value = record.getValue(joinOn.getLeftField());
                   values.add(value);
@@ -159,7 +146,7 @@ public class QueryAction
             {
                filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
 
-               for(QRecord record : queryOutput.getRecords())
+               for(QRecord record : queryOutputRecords)
                {
                   QQueryFilter subFilter = new QQueryFilter();
                   filter.addSubFilter(subFilter);
@@ -227,7 +214,7 @@ public class QueryAction
     ** not one created via List.of()).  This may include setting display values,
     ** translating possible values, and running post-record customizations.
     *******************************************************************************/
-   public void postRecordActions(List<QRecord> records)
+   public void postRecordActions(List<QRecord> records) throws QException
    {
       if(this.postQueryRecordCustomizer.isPresent())
       {
@@ -247,5 +234,10 @@ public class QueryAction
       {
          QValueFormatter.setDisplayValuesInRecords(queryInput.getTable(), records);
       }
+
+      if(queryInput.getIncludeAssociations())
+      {
+         manageAssociations(queryInput, records);
+      }
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java

@@ -31,6 +31,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.function.Consumer;
 import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@@ -61,7 +62,7 @@ public class CsvToQRecordAdapter
     ** using a given mapping.
     **
     *******************************************************************************/
-   public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer)
+   public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer) throws QException
    {
       buildRecordsFromCsv(new InputWrapper().withRecordPipe(recordPipe).withCsv(csv).withTable(table).withMapping(mapping).withRecordCustomizer(recordCustomizer));
    }
@@ -73,7 +74,7 @@ public class CsvToQRecordAdapter
     ** using a given mapping.
     **
     *******************************************************************************/
-   public List<QRecord> buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping)
+   public List<QRecord> buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping) throws QException
    {
       buildRecordsFromCsv(new InputWrapper().withCsv(csv).withTable(table).withMapping(mapping));
       return (recordList);
@@ -87,7 +88,7 @@ public class CsvToQRecordAdapter
     **
     ** todo - meta-data validation, type handling
     *******************************************************************************/
-   public void buildRecordsFromCsv(InputWrapper inputWrapper)
+   public void buildRecordsFromCsv(InputWrapper inputWrapper) throws QException
    {
       String                   csv              = inputWrapper.getCsv();
       AbstractQFieldMapping<?> mapping          = inputWrapper.getMapping();
@@ -297,7 +298,7 @@ public class CsvToQRecordAdapter
    /*******************************************************************************
     ** Add a record - either to the pipe, or list, whichever we're building.
     *******************************************************************************/
-   private void addRecord(QRecord record)
+   private void addRecord(QRecord record) throws QException
    {
       if(recordPipe != null)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
 
 import java.io.Serializable;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
@@ -63,7 +64,7 @@ public class QueryOutput extends AbstractActionOutput implements Serializable
     ** that could be read asynchronously, at any time, by another thread - SO - only
     ** completely populated records should be passed into this method.
     *******************************************************************************/
-   public void addRecord(QRecord record)
+   public void addRecord(QRecord record) throws QException
    {
       storage.addRecord(record);
    }
@@ -73,7 +74,7 @@ public class QueryOutput extends AbstractActionOutput implements Serializable
    /*******************************************************************************
     ** add a list of records to this output
     *******************************************************************************/
-   public void addRecords(List<QRecord> records)
+   public void addRecords(List<QRecord> records) throws QException
    {
       storage.addRecords(records);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
 
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
@@ -53,7 +54,7 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface
     ** add a record to this output
     *******************************************************************************/
    @Override
-   public void addRecord(QRecord record)
+   public void addRecord(QRecord record) throws QException
    {
       recordPipe.addRecord(record);
    }
@@ -64,7 +65,7 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface
     ** add a list of records to this output
     *******************************************************************************/
    @Override
-   public void addRecords(List<QRecord> records)
+   public void addRecords(List<QRecord> records) throws QException
    {
       recordPipe.addRecords(records);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputStorageInterface.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
 
 
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
@@ -36,13 +37,13 @@ interface QueryOutputStorageInterface
    /*******************************************************************************
     ** add a records to this output
     *******************************************************************************/
-   void addRecord(QRecord record);
+   void addRecord(QRecord record) throws QException;
 
 
    /*******************************************************************************
     ** add a list of records to this output
     *******************************************************************************/
-   void addRecords(List<QRecord> records);
+   void addRecords(List<QRecord> records) throws QException;
 
    /*******************************************************************************
     ** Get all stored records

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java

@@ -164,6 +164,42 @@ class QueryActionTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testQueryAssociationsWithPipe() throws QException
+   {
+      QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true);
+      insert2OrdersWith3Lines3LineExtrinsicsAnd4OrderExtrinsicAssociations();
+
+      RecordPipe pipe       = new RecordPipe();
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+      queryInput.setRecordPipe(pipe);
+      queryInput.setIncludeAssociations(true);
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+      assertNotNull(queryOutput);
+
+      List<QRecord> records = pipe.consumeAvailableRecords();
+      assertThat(records).isNotEmpty();
+
+      QRecord order0 = records.get(0);
+      assertEquals(2, order0.getAssociatedRecords().get("orderLine").size());
+      assertEquals(3, order0.getAssociatedRecords().get("extrinsics").size());
+
+      QRecord orderLine00 = order0.getAssociatedRecords().get("orderLine").get(0);
+      assertEquals(1, orderLine00.getAssociatedRecords().get("extrinsics").size());
+      QRecord orderLine01 = order0.getAssociatedRecords().get("orderLine").get(1);
+      assertEquals(2, orderLine01.getAssociatedRecords().get("extrinsics").size());
+
+      QRecord order1 = records.get(1);
+      assertEquals(1, order1.getAssociatedRecords().get("orderLine").size());
+      assertEquals(1, order1.getAssociatedRecords().get("extrinsics").size());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.adapters;
 
 import java.util.List;
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QIndexBasedFieldMapping;
 import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -48,7 +49,7 @@ class CsvToQRecordAdapterTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   public void test_buildRecordsFromCsv_nullInput()
+   public void test_buildRecordsFromCsv_nullInput() throws QException
    {
       testExpectedToThrow(null);
    }
@@ -59,7 +60,7 @@ class CsvToQRecordAdapterTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   public void test_buildRecordsFromCsv_emptyStringInput()
+   public void test_buildRecordsFromCsv_emptyStringInput() throws QException
    {
       testExpectedToThrow("");
    }
@@ -69,7 +70,7 @@ class CsvToQRecordAdapterTest extends BaseTest
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void testExpectedToThrow(String csv)
+   private void testExpectedToThrow(String csv) throws QException
    {
       try
       {
@@ -92,7 +93,7 @@ class CsvToQRecordAdapterTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   public void test_buildRecordsFromCsv_emptyList()
+   public void test_buildRecordsFromCsv_emptyList() throws QException
    {
       CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
       List<QRecord>       qRecords            = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader(), TestUtils.defineTablePerson(), null);
@@ -142,7 +143,7 @@ class CsvToQRecordAdapterTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   public void test_buildRecordsFromCsv_oneRowStandardHeaderNoMapping()
+   public void test_buildRecordsFromCsv_oneRowStandardHeaderNoMapping() throws QException
    {
       CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
       List<QRecord>       qRecords            = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1(), TestUtils.defineTablePerson(), null);
@@ -159,7 +160,7 @@ class CsvToQRecordAdapterTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   public void test_buildRecordsFromCsv_twoRowsStandardHeaderNoMapping()
+   public void test_buildRecordsFromCsv_twoRowsStandardHeaderNoMapping() throws QException
    {
       CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
       List<QRecord>       qRecords            = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1() + getPersonCsvRow2(), TestUtils.defineTablePerson(), null);
@@ -179,7 +180,7 @@ class CsvToQRecordAdapterTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   public void test_buildRecordsFromCsv_oneRowCustomKeyBasedMapping()
+   public void test_buildRecordsFromCsv_oneRowCustomKeyBasedMapping() throws QException
    {
       String csvCustomHeader = """
          "id","created","modified","first","last","birthday","email"\r
@@ -209,7 +210,7 @@ class CsvToQRecordAdapterTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   public void test_buildRecordsFromCsv_twoRowsCustomIndexBasedMapping()
+   public void test_buildRecordsFromCsv_twoRowsCustomIndexBasedMapping() throws QException
    {
       int index = 1;
       QIndexBasedFieldMapping mapping = new QIndexBasedFieldMapping()
@@ -241,7 +242,7 @@ class CsvToQRecordAdapterTest extends BaseTest
     ** header names on the RHS.
     *******************************************************************************/
    @Test
-   public void test_duplicatedColumnHeaders()
+   public void test_duplicatedColumnHeaders() throws QException
    {
       QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping()
          .withMapping("id", "id")
@@ -291,7 +292,7 @@ class CsvToQRecordAdapterTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   void testByteOrderMarker()
+   void testByteOrderMarker() throws QException
    {
       CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
 
@@ -313,7 +314,7 @@ class CsvToQRecordAdapterTest extends BaseTest
     ** Fix an IndexOutOfBounds that we used to throw.
     *******************************************************************************/
    @Test
-   void testTooFewBodyColumns()
+   void testTooFewBodyColumns() throws QException
    {
       CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
       List<QRecord> records = csvToQRecordAdapter.buildRecordsFromCsv("""
@@ -331,7 +332,7 @@ class CsvToQRecordAdapterTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   public void testTooFewColumnsIndexMapping()
+   public void testTooFewColumnsIndexMapping() throws QException
    {
       int index = 1;
       QIndexBasedFieldMapping mapping = new QIndexBasedFieldMapping()
@@ -353,7 +354,7 @@ class CsvToQRecordAdapterTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   void testCaseSensitiveHeaders()
+   void testCaseSensitiveHeaders() throws QException
    {
       CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
       csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
@@ -376,7 +377,7 @@ class CsvToQRecordAdapterTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   void testCaseInsensitiveHeaders()
+   void testCaseInsensitiveHeaders() throws QException
    {
       CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
       csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcessTest.java

@@ -222,7 +222,10 @@ class TableSyncProcessTest extends BaseTest
       public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
       {
          List<QRecord> qRecords = TestUtils.queryTable(QContext.getQInstance(), TestUtils.TABLE_NAME_PERSON_MEMORY);
-         qRecords.forEach(r -> getRecordPipe().addRecord(r));
+         for(QRecord qRecord : qRecords)
+         {
+            getRecordPipe().addRecord(qRecord);
+         }
 
          ////////////////////////////////////////
          // re-add records 1 and 5 to the pipe //

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java

@@ -50,6 +50,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -87,7 +88,7 @@ public class QueryManager
       /*******************************************************************************
        **
        *******************************************************************************/
-      void processResultSet(ResultSet rs) throws SQLException;
+      void processResultSet(ResultSet rs) throws SQLException, QException;
    }
 
 
@@ -95,7 +96,7 @@ public class QueryManager
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static void executeStatement(Connection connection, String sql, ResultSetProcessor processor, Object... params) throws SQLException
+   public static void executeStatement(Connection connection, String sql, ResultSetProcessor processor, Object... params) throws SQLException, QException
    {
       PreparedStatement statement = null;
       try
@@ -118,7 +119,7 @@ public class QueryManager
     ** Let the caller provide their own prepared statement (e.g., possibly with some
     ** customized settings/optimizations).
     *******************************************************************************/
-   public static void executeStatement(PreparedStatement statement, ResultSetProcessor processor, Object... params) throws SQLException
+   public static void executeStatement(PreparedStatement statement, ResultSetProcessor processor, Object... params) throws SQLException, QException
    {
       ResultSet resultSet = null;