QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-881 - Add queryJoins when rendering savedReports - tested in RDBMS module

Browse Source
This commit is contained in:
darin.kelkhoff
2024-03-28 15:44:46 -05:00
parent 1554815fd0
commit d0de637dee
4 changed files with 269 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java
View File

@ -23,8 +23,10 @@ package com.kingsrook.qqq.backend.core.processes.implementations.savedreports;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.core.type.TypeReference;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -32,14 +34,17 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -63,6 +68,8 @@ public class SavedReportToReportMetaDataAdapter
{
try
{
QInstance qInstance = QContext.getQInstance();
QReportMetaData reportMetaData = new QReportMetaData();
reportMetaData.setLabel(savedReport.getLabel());
@ -73,15 +80,11 @@ public class SavedReportToReportMetaDataAdapter
reportMetaData.setDataSources(List.of(dataSource));
dataSource.setName("main");
QTableMetaData table = QContext.getQInstance().getTable(savedReport.getTableName());
QTableMetaData table = qInstance.getTable(savedReport.getTableName());
dataSource.setSourceTable(savedReport.getTableName());
dataSource.setQueryFilter(JsonUtils.toObject(savedReport.getQueryFilterJson(), QQueryFilter.class));
// todo!!! oh my.
List<QueryJoin> queryJoins = null;
dataSource.setQueryJoins(queryJoins);
//////////////////////////
// set up the main view //
//////////////////////////
@ -103,10 +106,13 @@ public class SavedReportToReportMetaDataAdapter
///////////////////////////////////////////////////////////////////////////////////////////////////////////
// columns in the saved-report look like a JSON object, w/ a key "columns", which is an array of objects //
///////////////////////////////////////////////////////////////////////////////////////////////////////////
Set<String> neededJoinTables = new HashSet<>();
List<QReportField> reportColumns = new ArrayList<>();
view.setColumns(reportColumns);
Map<String, Object> columnsObject = JsonUtils.toObject(savedReport.getColumnsJson(), new TypeReference<>() {});
List<Map<String, Object>> columns = (List<Map<String, Object>>) columnsObject.get("columns");
List<QReportField> reportColumns = new ArrayList<>();
for(Map<String, Object> column : columns)
{
if(column.containsKey("isVisible") && !"true".equals(ValueUtils.getValueAsString(column.get("isVisible"))))
@ -114,11 +120,28 @@ public class SavedReportToReportMetaDataAdapter
continue;
}
QFieldMetaData field = null;
String fieldName = ValueUtils.getValueAsString(column.get("name"));
QFieldMetaData field;
String fieldName = ValueUtils.getValueAsString(column.get("name"));
if(fieldName.contains("."))
{
// todo - join!
String joinTableName = fieldName.replaceAll("\\..*", "");
String joinFieldName = fieldName.replaceAll(".*\\.", "");
QTableMetaData joinTable = qInstance.getTable(joinTableName);
if(joinTable == null)
{
LOG.warn("Saved Report has an unrecognized join table name", logPair("savedReportId", savedReport.getId()), logPair("joinTable", joinTable), logPair("fieldName", fieldName));
continue;
}
neededJoinTables.add(joinTableName);
field = joinTable.getFields().get(joinFieldName);
if(field == null)
{
LOG.warn("Saved Report has an unrecognized join field name", logPair("savedReportId", savedReport.getId()), logPair("fieldName", fieldName));
continue;
}
}
else
{
@ -148,7 +171,40 @@ public class SavedReportToReportMetaDataAdapter
}
}
view.setColumns(reportColumns);
///////////////////////////////////////////////////////////////////////////////////////////
// set up joins, if we need any //
// note - test coverage here is provided by RDBMS module's GenerateReportActionRDBMSTest //
///////////////////////////////////////////////////////////////////////////////////////////
if(!neededJoinTables.isEmpty())
{
List<QueryJoin> queryJoins = new ArrayList<>();
dataSource.setQueryJoins(queryJoins);
for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
{
if(neededJoinTables.contains(exposedJoin.getJoinTable()))
{
QueryJoin queryJoin = new QueryJoin(exposedJoin.getJoinTable())
.withSelect(true)
.withType(QueryJoin.Type.LEFT)
.withBaseTableOrAlias(null)
.withAlias(null);
if(exposedJoin.getJoinPath().size() == 1)
{
// this is similar logic that QFMD has
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
// todo - what about a join with a longer path? it would be nice to pass such joinNames through there too, //
// but what, that would actually be multiple queryJoins? needs a fair amount of thought. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
queryJoin.setJoinMetaData(qInstance.getJoin(exposedJoin.getJoinPath().get(0)));
}
queryJoins.add(queryJoin);
}
}
}
///////////////////////////////////////////////
// if it's a pivot report, add that view too //

11
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java
View File

@ -954,6 +954,17 @@ public abstract class AbstractRDBMSAction
/*******************************************************************************
** Make it easy (e.g., for tests) to turn on logging of SQL
*******************************************************************************/
public static void setLogSQL(boolean on, boolean doReformat, String loggerOrSystemOut)
{
setLogSQL(on);
setLogSQLOutput(loggerOrSystemOut);
setLogSQLReformat(doReformat);
}
/*******************************************************************************
** Make it easy (e.g., for tests) to turn on logging of SQL
*******************************************************************************/

17
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java
View File

@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@ -46,6 +47,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager;
@ -61,6 +63,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
public class TestUtils
{
public static final String DEFAULT_BACKEND_NAME = "default";
public static final String MEMORY_BACKEND_NAME = "memory";
public static final String TABLE_NAME_PERSON = "personTable";
public static final String TABLE_NAME_PERSONAL_ID_CARD = "personalIdCard";
@ -107,6 +110,7 @@ public class TestUtils
{
QInstance qInstance = new QInstance();
qInstance.addBackend(defineBackend());
qInstance.addBackend(defineMemoryBackend());
qInstance.addTable(defineTablePerson());
qInstance.addPossibleValueSource(definePvsPerson());
qInstance.addTable(defineTablePersonalIdCard());
@ -118,6 +122,18 @@ public class TestUtils
/*******************************************************************************
** Define the in-memory backend used in standard tests
*******************************************************************************/
public static QBackendMetaData defineMemoryBackend()
{
return new QBackendMetaData()
.withName(MEMORY_BACKEND_NAME)
.withBackendType(MemoryBackendModule.class);
}
/*******************************************************************************
** Define the authentication used in standard tests - using 'mock' type.
**
@ -243,6 +259,7 @@ public class TestUtils
.withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId"))
.withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_ORDER_LINE).withJoinName("orderJoinOrderLine"))
.withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ITEM).withJoinPath(List.of("orderJoinOrderLine", "orderLineJoinItem")))
.withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER_INSTRUCTIONS).withJoinPath(List.of("orderJoinCurrentOrderInstructions")).withLabel("Current Order Instructions"))
.withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE))
.withField(new QFieldMetaData("billToPersonId", QFieldType.INTEGER).withBackendName("bill_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON))
.withField(new QFieldMetaData("shipToPersonId", QFieldType.INTEGER).withBackendName("ship_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON))

174
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java
View File

@ -23,26 +23,42 @@ package com.kingsrook.qqq.backend.module.rdbms.reporting;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType;
import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport;
import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.RenderSavedReportMetaDataProducer;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.module.rdbms.TestUtils;
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -196,6 +212,164 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest
}
/*******************************************************************************
**
*******************************************************************************/
private List<String> runSavedReportForCSV(SavedReport newSavedReport) throws Exception
{
newSavedReport.setLabel("Test Report");
QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, null);
QRecord savedReport = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(newSavedReport)).getRecords().get(0);
RunProcessInput input = new RunProcessInput();
input.setProcessName(RenderSavedReportMetaDataProducer.NAME);
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
input.setCallback(QProcessCallbackFactory.forRecord(savedReport));
input.addValue("reportFormat", ReportFormatPossibleValueEnum.CSV.getPossibleValueId());
RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
return (FileUtils.readLines(new File(runProcessOutput.getValueString("serverFilePath")), StandardCharsets.UTF_8));
}
/*******************************************************************************
** in here, by potentially ambiguous, we mean where there are possible joins
** between the order and orderInstructions tables.
*******************************************************************************/
@Test
void testSavedReportWithPotentiallyAmbiguousExposedJoinSelections() throws Exception
{
List<String> lines = runSavedReportForCSV(new SavedReport()
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withColumnsJson("""
{"columns":[
{"name": "id"},
{"name": "storeId"},
{"name": "orderInstructions.instructions"}
]}""")
.withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())));
assertEquals("""
"Id","Store","Instructions"
""".trim(), lines.get(0));
assertEquals("""
"1","Q-Mart","order 1 v2"
""".trim(), lines.get(1));
}
/*******************************************************************************
** in here, by potentially ambiguous, we mean where there are possible joins
** between the order and orderInstructions tables.
*******************************************************************************/
@Test
void testSavedReportWithPotentiallyAmbiguousExposedJoinSelectedAndOrdered() throws Exception
{
List<String> lines = runSavedReportForCSV(new SavedReport()
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withColumnsJson("""
{"columns":[
{"name": "id"},
{"name": "storeId"},
{"name": "orderInstructions.instructions"}
]}""")
.withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()
.withOrderBy(new QFilterOrderBy("orderInstructions.id", false))
)));
assertEquals("""
"Id","Store","Instructions"
""".trim(), lines.get(0));
assertEquals("""
"8","QDepot","order 8 v1"
""".trim(), lines.get(1));
}
/*******************************************************************************
** in here, by potentially ambiguous, we mean where there are possible joins
** between the order and orderInstructions tables.
*******************************************************************************/
@Test
void testSavedReportWithPotentiallyAmbiguousExposedJoinCriteria() throws Exception
{
List<String> lines = runSavedReportForCSV(new SavedReport()
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withColumnsJson("""
{"columns":[
{"name": "id"},
{"name": "storeId"}
]}""")
.withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()
.withCriteria(new QFilterCriteria("orderInstructions.instructions", QCriteriaOperator.CONTAINS, "v3"))
)));
assertEquals("""
"Id","Store"
""".trim(), lines.get(0));
assertEquals("""
"2","Q-Mart"
""".trim(), lines.get(1));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSavedReportWithExposedJoinMultipleTablesAwaySelected() throws Exception
{
List<String> lines = runSavedReportForCSV(new SavedReport()
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withColumnsJson("""
{"columns":[
{"name": "id"},
{"name": "storeId"},
{"name": "item.description"}
]}""")
.withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())));
assertEquals("""
"Id","Store","Description"
""".trim(), lines.get(0));
assertEquals("""
"1","Q-Mart","Q-Mart Item 1"
""".trim(), lines.get(1));
}
// todo - similar to above, but w/o selecting, only filtering
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSavedReportWithExposedJoinMultipleTablesAwayAsCriteria() throws Exception
{
List<String> lines = runSavedReportForCSV(new SavedReport()
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withColumnsJson("""
{"columns":[
{"name": "id"},
{"name": "storeId"}
]}""")
.withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()
.withCriteria(new QFilterCriteria("item.description", QCriteriaOperator.CONTAINS, "Item 7"))
)));
assertEquals("""
"Id","Store"
""".trim(), lines.get(0));
assertEquals("""
"6","QDepot"
""".trim(), lines.get(1));
}
/*******************************************************************************
**