QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update to allow includeAssociations when querying into a record pipe. This meant propagating a lot of exceptions...

Browse Source
This commit is contained in:
darin.kelkhoff
2023-04-28 12:14:12 -05:00
parent 6f99111c52
commit b7e39d6953
11 changed files with 93 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/BufferedRecordPipe.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -63,7 +64,7 @@ public class BufferedRecordPipe extends RecordPipe
**
*******************************************************************************/
@Override
public void addRecord(QRecord record)
public void addRecord(QRecord record) throws QException
{
buffer.add(record);
if(buffer.size() >= bufferSize)
@ -78,7 +79,7 @@ public class BufferedRecordPipe extends RecordPipe
/*******************************************************************************
**
*******************************************************************************/
public void finalFlush()
public void finalFlush() throws QException
{
if(!buffer.isEmpty())
{

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java
View File

@ -26,10 +26,11 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeConsumer;
/*******************************************************************************
@ -47,7 +48,7 @@ public class RecordPipe
private boolean isTerminated = false;
private Consumer<List<QRecord>> postRecordActions = null;
private UnsafeConsumer<List<QRecord>, QException> postRecordActions = null;
/////////////////////////////////////
// See usage below for explanation //
@ -93,7 +94,7 @@ public class RecordPipe
/*******************************************************************************
** Add a record to the pipe. Will block if the pipe is full. Will noop if pipe is terminated.
*******************************************************************************/
public void addRecord(QRecord record)
public void addRecord(QRecord record) throws QException
{
if(isTerminated)
{
@ -109,7 +110,7 @@ public class RecordPipe
// (which we'll create as a field in this class, to avoid always re-constructing) //
////////////////////////////////////////////////////////////////////////////////////
singleRecordListForPostRecordActions.add(record);
postRecordActions.accept(singleRecordListForPostRecordActions);
postRecordActions.run(singleRecordListForPostRecordActions);
record = singleRecordListForPostRecordActions.remove(0);
}
@ -152,11 +153,11 @@ public class RecordPipe
/*******************************************************************************
** Add a list of records to the pipe. Will block if the pipe is full. Will noop if pipe is terminated.
*******************************************************************************/
public void addRecords(List<QRecord> records)
public void addRecords(List<QRecord> records) throws QException
{
if(postRecordActions != null)
{
postRecordActions.accept(records);
postRecordActions.run(records);
}
//////////////////////////////////////////////////////////////////////////////////////////////////
@ -207,7 +208,7 @@ public class RecordPipe
/*******************************************************************************
**
*******************************************************************************/
public void setPostRecordActions(Consumer<List<QRecord>> postRecordActions)
public void setPostRecordActions(UnsafeConsumer<List<QRecord>, QException> postRecordActions)
{
this.postRecordActions = postRecordActions;
}

26
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java
View File

@ -85,14 +85,6 @@ public class QueryAction
queryInput.getRecordPipe().setPostRecordActions(this::postRecordActions);
}
if(queryInput.getIncludeAssociations() && queryInput.getRecordPipe() != null)
{
//////////////////////////////////////////////
// todo - support this in the future maybe? //
//////////////////////////////////////////////
throw (new QException("Associations may not be fetched into a RecordPipe."));
}
QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(queryInput.getBackend());
// todo pre-customization - just get to modify the request?
@ -109,11 +101,6 @@ public class QueryAction
postRecordActions(queryOutput.getRecords());
}
if(queryInput.getIncludeAssociations())
{
manageAssociations(queryInput, queryOutput);
}
return queryOutput;
}
@ -122,7 +109,7 @@ public class QueryAction
/*******************************************************************************
**
*******************************************************************************/
private void manageAssociations(QueryInput queryInput, QueryOutput queryOutput) throws QException
private void manageAssociations(QueryInput queryInput, List<QRecord> queryOutputRecords) throws QException
{
QTableMetaData table = queryInput.getTable();
for(Association association : CollectionUtils.nonNullList(table.getAssociations()))
@ -147,7 +134,7 @@ public class QueryAction
{
JoinOn joinOn = join.getJoinOns().get(0);
Set<Serializable> values = new HashSet<>();
for(QRecord record : queryOutput.getRecords())
for(QRecord record : queryOutputRecords)
{
Serializable value = record.getValue(joinOn.getLeftField());
values.add(value);
@ -159,7 +146,7 @@ public class QueryAction
{
filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
for(QRecord record : queryOutput.getRecords())
for(QRecord record : queryOutputRecords)
{
QQueryFilter subFilter = new QQueryFilter();
filter.addSubFilter(subFilter);
@ -227,7 +214,7 @@ public class QueryAction
** not one created via List.of()). This may include setting display values,
** translating possible values, and running post-record customizations.
*******************************************************************************/
public void postRecordActions(List<QRecord> records)
public void postRecordActions(List<QRecord> records) throws QException
{
if(this.postQueryRecordCustomizer.isPresent())
{
@ -247,5 +234,10 @@ public class QueryAction
{
QValueFormatter.setDisplayValuesInRecords(queryInput.getTable(), records);
}
if(queryInput.getIncludeAssociations())
{
manageAssociations(queryInput, records);
}
}
}

9
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java
View File

@ -31,6 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@ -61,7 +62,7 @@ public class CsvToQRecordAdapter
** using a given mapping.
**
*******************************************************************************/
public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer)
public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer) throws QException
{
buildRecordsFromCsv(new InputWrapper().withRecordPipe(recordPipe).withCsv(csv).withTable(table).withMapping(mapping).withRecordCustomizer(recordCustomizer));
}
@ -73,7 +74,7 @@ public class CsvToQRecordAdapter
** using a given mapping.
**
*******************************************************************************/
public List<QRecord> buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping)
public List<QRecord> buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping) throws QException
{
buildRecordsFromCsv(new InputWrapper().withCsv(csv).withTable(table).withMapping(mapping));
return (recordList);
@ -87,7 +88,7 @@ public class CsvToQRecordAdapter
**
** todo - meta-data validation, type handling
*******************************************************************************/
public void buildRecordsFromCsv(InputWrapper inputWrapper)
public void buildRecordsFromCsv(InputWrapper inputWrapper) throws QException
{
String csv = inputWrapper.getCsv();
AbstractQFieldMapping<?> mapping = inputWrapper.getMapping();
@ -297,7 +298,7 @@ public class CsvToQRecordAdapter
/*******************************************************************************
** Add a record - either to the pipe, or list, whichever we're building.
*******************************************************************************/
private void addRecord(QRecord record)
private void addRecord(QRecord record) throws QException
{
if(recordPipe != null)
{

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
import java.io.Serializable;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -63,7 +64,7 @@ public class QueryOutput extends AbstractActionOutput implements Serializable
** that could be read asynchronously, at any time, by another thread - SO - only
** completely populated records should be passed into this method.
*******************************************************************************/
public void addRecord(QRecord record)
public void addRecord(QRecord record) throws QException
{
storage.addRecord(record);
}
@ -73,7 +74,7 @@ public class QueryOutput extends AbstractActionOutput implements Serializable
/*******************************************************************************
** add a list of records to this output
*******************************************************************************/
public void addRecords(List<QRecord> records)
public void addRecords(List<QRecord> records) throws QException
{
storage.addRecords(records);
}

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -53,7 +54,7 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface
** add a record to this output
*******************************************************************************/
@Override
public void addRecord(QRecord record)
public void addRecord(QRecord record) throws QException
{
recordPipe.addRecord(record);
}
@ -64,7 +65,7 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface
** add a list of records to this output
*******************************************************************************/
@Override
public void addRecords(List<QRecord> records)
public void addRecords(List<QRecord> records) throws QException
{
recordPipe.addRecords(records);
}

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputStorageInterface.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -36,13 +37,13 @@ interface QueryOutputStorageInterface
/*******************************************************************************
** add a records to this output
*******************************************************************************/
void addRecord(QRecord record);
void addRecord(QRecord record) throws QException;
/*******************************************************************************
** add a list of records to this output
*******************************************************************************/
void addRecords(List<QRecord> records);
void addRecords(List<QRecord> records) throws QException;
/*******************************************************************************
** Get all stored records

36
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java
View File

@ -164,6 +164,42 @@ class QueryActionTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testQueryAssociationsWithPipe() throws QException
{
QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true);
insert2OrdersWith3Lines3LineExtrinsicsAnd4OrderExtrinsicAssociations();
RecordPipe pipe = new RecordPipe();
QueryInput queryInput = new QueryInput();
queryInput.setTableName(TestUtils.TABLE_NAME_ORDER);
queryInput.setRecordPipe(pipe);
queryInput.setIncludeAssociations(true);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertNotNull(queryOutput);
List<QRecord> records = pipe.consumeAvailableRecords();
assertThat(records).isNotEmpty();
QRecord order0 = records.get(0);
assertEquals(2, order0.getAssociatedRecords().get("orderLine").size());
assertEquals(3, order0.getAssociatedRecords().get("extrinsics").size());
QRecord orderLine00 = order0.getAssociatedRecords().get("orderLine").get(0);
assertEquals(1, orderLine00.getAssociatedRecords().get("extrinsics").size());
QRecord orderLine01 = order0.getAssociatedRecords().get("orderLine").get(1);
assertEquals(2, orderLine01.getAssociatedRecords().get("extrinsics").size());
QRecord order1 = records.get(1);
assertEquals(1, order1.getAssociatedRecords().get("orderLine").size());
assertEquals(1, order1.getAssociatedRecords().get("extrinsics").size());
}
/*******************************************************************************
**
*******************************************************************************/

29
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.adapters;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QIndexBasedFieldMapping;
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -48,7 +49,7 @@ class CsvToQRecordAdapterTest extends BaseTest
**
*******************************************************************************/
@Test
public void test_buildRecordsFromCsv_nullInput()
public void test_buildRecordsFromCsv_nullInput() throws QException
{
testExpectedToThrow(null);
}
@ -59,7 +60,7 @@ class CsvToQRecordAdapterTest extends BaseTest
**
*******************************************************************************/
@Test
public void test_buildRecordsFromCsv_emptyStringInput()
public void test_buildRecordsFromCsv_emptyStringInput() throws QException
{
testExpectedToThrow("");
}
@ -69,7 +70,7 @@ class CsvToQRecordAdapterTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
private void testExpectedToThrow(String csv)
private void testExpectedToThrow(String csv) throws QException
{
try
{
@ -92,7 +93,7 @@ class CsvToQRecordAdapterTest extends BaseTest
**
*******************************************************************************/
@Test
public void test_buildRecordsFromCsv_emptyList()
public void test_buildRecordsFromCsv_emptyList() throws QException
{
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader(), TestUtils.defineTablePerson(), null);
@ -142,7 +143,7 @@ class CsvToQRecordAdapterTest extends BaseTest
**
*******************************************************************************/
@Test
public void test_buildRecordsFromCsv_oneRowStandardHeaderNoMapping()
public void test_buildRecordsFromCsv_oneRowStandardHeaderNoMapping() throws QException
{
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1(), TestUtils.defineTablePerson(), null);
@ -159,7 +160,7 @@ class CsvToQRecordAdapterTest extends BaseTest
**
*******************************************************************************/
@Test
public void test_buildRecordsFromCsv_twoRowsStandardHeaderNoMapping()
public void test_buildRecordsFromCsv_twoRowsStandardHeaderNoMapping() throws QException
{
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1() + getPersonCsvRow2(), TestUtils.defineTablePerson(), null);
@ -179,7 +180,7 @@ class CsvToQRecordAdapterTest extends BaseTest
**
*******************************************************************************/
@Test
public void test_buildRecordsFromCsv_oneRowCustomKeyBasedMapping()
public void test_buildRecordsFromCsv_oneRowCustomKeyBasedMapping() throws QException
{
String csvCustomHeader = """
"id","created","modified","first","last","birthday","email"\r
@ -209,7 +210,7 @@ class CsvToQRecordAdapterTest extends BaseTest
**
*******************************************************************************/
@Test
public void test_buildRecordsFromCsv_twoRowsCustomIndexBasedMapping()
public void test_buildRecordsFromCsv_twoRowsCustomIndexBasedMapping() throws QException
{
int index = 1;
QIndexBasedFieldMapping mapping = new QIndexBasedFieldMapping()
@ -241,7 +242,7 @@ class CsvToQRecordAdapterTest extends BaseTest
** header names on the RHS.
*******************************************************************************/
@Test
public void test_duplicatedColumnHeaders()
public void test_duplicatedColumnHeaders() throws QException
{
QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping()
.withMapping("id", "id")
@ -291,7 +292,7 @@ class CsvToQRecordAdapterTest extends BaseTest
**
*******************************************************************************/
@Test
void testByteOrderMarker()
void testByteOrderMarker() throws QException
{
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
@ -313,7 +314,7 @@ class CsvToQRecordAdapterTest extends BaseTest
** Fix an IndexOutOfBounds that we used to throw.
*******************************************************************************/
@Test
void testTooFewBodyColumns()
void testTooFewBodyColumns() throws QException
{
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
List<QRecord> records = csvToQRecordAdapter.buildRecordsFromCsv("""
@ -331,7 +332,7 @@ class CsvToQRecordAdapterTest extends BaseTest
**
*******************************************************************************/
@Test
public void testTooFewColumnsIndexMapping()
public void testTooFewColumnsIndexMapping() throws QException
{
int index = 1;
QIndexBasedFieldMapping mapping = new QIndexBasedFieldMapping()
@ -353,7 +354,7 @@ class CsvToQRecordAdapterTest extends BaseTest
**
*******************************************************************************/
@Test
void testCaseSensitiveHeaders()
void testCaseSensitiveHeaders() throws QException
{
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
@ -376,7 +377,7 @@ class CsvToQRecordAdapterTest extends BaseTest
**
*******************************************************************************/
@Test
void testCaseInsensitiveHeaders()
void testCaseInsensitiveHeaders() throws QException
{
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()

5
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcessTest.java
View File

@ -222,7 +222,10 @@ class TableSyncProcessTest extends BaseTest
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
List<QRecord> qRecords = TestUtils.queryTable(QContext.getQInstance(), TestUtils.TABLE_NAME_PERSON_MEMORY);
qRecords.forEach(r -> getRecordPipe().addRecord(r));
for(QRecord qRecord : qRecords)
{
getRecordPipe().addRecord(qRecord);
}
////////////////////////////////////////
// re-add records 1 and 5 to the pipe //

7
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java
View File

@ -50,6 +50,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -87,7 +88,7 @@ public class QueryManager
/*******************************************************************************
**
*******************************************************************************/
void processResultSet(ResultSet rs) throws SQLException;
void processResultSet(ResultSet rs) throws SQLException, QException;
}
@ -95,7 +96,7 @@ public class QueryManager
/*******************************************************************************
**
*******************************************************************************/
public static void executeStatement(Connection connection, String sql, ResultSetProcessor processor, Object... params) throws SQLException
public static void executeStatement(Connection connection, String sql, ResultSetProcessor processor, Object... params) throws SQLException, QException
{
PreparedStatement statement = null;
try
@ -118,7 +119,7 @@ public class QueryManager
** Let the caller provide their own prepared statement (e.g., possibly with some
** customized settings/optimizations).
*******************************************************************************/
public static void executeStatement(PreparedStatement statement, ResultSetProcessor processor, Object... params) throws SQLException
public static void executeStatement(PreparedStatement statement, ResultSetProcessor processor, Object... params) throws SQLException, QException
{
ResultSet resultSet = null;