From d0de637dee3d2c7b096a55f47acc0f41e9cff06a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 28 Mar 2024 15:44:46 -0500 Subject: [PATCH] CE-881 - Add queryJoins when rendering savedReports - tested in RDBMS module --- .../SavedReportToReportMetaDataAdapter.java | 78 ++++++-- .../rdbms/actions/AbstractRDBMSAction.java | 11 ++ .../qqq/backend/module/rdbms/TestUtils.java | 17 ++ .../GenerateReportActionRDBMSTest.java | 174 ++++++++++++++++++ 4 files changed, 269 insertions(+), 11 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java index 5833bf86..431349ef 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java @@ -23,8 +23,10 @@ package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import com.fasterxml.jackson.core.type.TypeReference; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -32,14 +34,17 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -63,6 +68,8 @@ public class SavedReportToReportMetaDataAdapter { try { + QInstance qInstance = QContext.getQInstance(); + QReportMetaData reportMetaData = new QReportMetaData(); reportMetaData.setLabel(savedReport.getLabel()); @@ -73,15 +80,11 @@ public class SavedReportToReportMetaDataAdapter reportMetaData.setDataSources(List.of(dataSource)); dataSource.setName("main"); - QTableMetaData table = QContext.getQInstance().getTable(savedReport.getTableName()); + QTableMetaData table = qInstance.getTable(savedReport.getTableName()); dataSource.setSourceTable(savedReport.getTableName()); dataSource.setQueryFilter(JsonUtils.toObject(savedReport.getQueryFilterJson(), QQueryFilter.class)); - // todo!!! oh my. - List queryJoins = null; - dataSource.setQueryJoins(queryJoins); - ////////////////////////// // set up the main view // ////////////////////////// @@ -103,10 +106,13 @@ public class SavedReportToReportMetaDataAdapter /////////////////////////////////////////////////////////////////////////////////////////////////////////// // columns in the saved-report look like a JSON object, w/ a key "columns", which is an array of objects // /////////////////////////////////////////////////////////////////////////////////////////////////////////// + Set neededJoinTables = new HashSet<>(); + + List reportColumns = new ArrayList<>(); + view.setColumns(reportColumns); + Map columnsObject = JsonUtils.toObject(savedReport.getColumnsJson(), new TypeReference<>() {}); List> columns = (List>) columnsObject.get("columns"); - List reportColumns = new ArrayList<>(); - for(Map column : columns) { if(column.containsKey("isVisible") && !"true".equals(ValueUtils.getValueAsString(column.get("isVisible")))) @@ -114,11 +120,28 @@ public class SavedReportToReportMetaDataAdapter continue; } - QFieldMetaData field = null; - String fieldName = ValueUtils.getValueAsString(column.get("name")); + QFieldMetaData field; + String fieldName = ValueUtils.getValueAsString(column.get("name")); if(fieldName.contains(".")) { - // todo - join! + String joinTableName = fieldName.replaceAll("\\..*", ""); + String joinFieldName = fieldName.replaceAll(".*\\.", ""); + + QTableMetaData joinTable = qInstance.getTable(joinTableName); + if(joinTable == null) + { + LOG.warn("Saved Report has an unrecognized join table name", logPair("savedReportId", savedReport.getId()), logPair("joinTable", joinTable), logPair("fieldName", fieldName)); + continue; + } + + neededJoinTables.add(joinTableName); + + field = joinTable.getFields().get(joinFieldName); + if(field == null) + { + LOG.warn("Saved Report has an unrecognized join field name", logPair("savedReportId", savedReport.getId()), logPair("fieldName", fieldName)); + continue; + } } else { @@ -148,7 +171,40 @@ public class SavedReportToReportMetaDataAdapter } } - view.setColumns(reportColumns); + /////////////////////////////////////////////////////////////////////////////////////////// + // set up joins, if we need any // + // note - test coverage here is provided by RDBMS module's GenerateReportActionRDBMSTest // + /////////////////////////////////////////////////////////////////////////////////////////// + if(!neededJoinTables.isEmpty()) + { + List queryJoins = new ArrayList<>(); + dataSource.setQueryJoins(queryJoins); + + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + if(neededJoinTables.contains(exposedJoin.getJoinTable())) + { + QueryJoin queryJoin = new QueryJoin(exposedJoin.getJoinTable()) + .withSelect(true) + .withType(QueryJoin.Type.LEFT) + .withBaseTableOrAlias(null) + .withAlias(null); + + if(exposedJoin.getJoinPath().size() == 1) + { + // this is similar logic that QFMD has + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // todo - what about a join with a longer path? it would be nice to pass such joinNames through there too, // + // but what, that would actually be multiple queryJoins? needs a fair amount of thought. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + queryJoin.setJoinMetaData(qInstance.getJoin(exposedJoin.getJoinPath().get(0))); + } + + queryJoins.add(queryJoin); + } + } + } /////////////////////////////////////////////// // if it's a pivot report, add that view too // diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index fccc54c7..ac62386f 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -954,6 +954,17 @@ public abstract class AbstractRDBMSAction + /******************************************************************************* + ** Make it easy (e.g., for tests) to turn on logging of SQL + *******************************************************************************/ + public static void setLogSQL(boolean on, boolean doReformat, String loggerOrSystemOut) + { + setLogSQL(on); + setLogSQLOutput(loggerOrSystemOut); + setLogSQLReformat(doReformat); + } + + /******************************************************************************* ** Make it easy (e.g., for tests) to turn on logging of SQL *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java index d882807a..2570f6ea 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -46,6 +47,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; @@ -61,6 +63,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; public class TestUtils { public static final String DEFAULT_BACKEND_NAME = "default"; + public static final String MEMORY_BACKEND_NAME = "memory"; public static final String TABLE_NAME_PERSON = "personTable"; public static final String TABLE_NAME_PERSONAL_ID_CARD = "personalIdCard"; @@ -107,6 +110,7 @@ public class TestUtils { QInstance qInstance = new QInstance(); qInstance.addBackend(defineBackend()); + qInstance.addBackend(defineMemoryBackend()); qInstance.addTable(defineTablePerson()); qInstance.addPossibleValueSource(definePvsPerson()); qInstance.addTable(defineTablePersonalIdCard()); @@ -118,6 +122,18 @@ public class TestUtils + /******************************************************************************* + ** Define the in-memory backend used in standard tests + *******************************************************************************/ + public static QBackendMetaData defineMemoryBackend() + { + return new QBackendMetaData() + .withName(MEMORY_BACKEND_NAME) + .withBackendType(MemoryBackendModule.class); + } + + + /******************************************************************************* ** Define the authentication used in standard tests - using 'mock' type. ** @@ -243,6 +259,7 @@ public class TestUtils .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId")) .withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_ORDER_LINE).withJoinName("orderJoinOrderLine")) .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ITEM).withJoinPath(List.of("orderJoinOrderLine", "orderLineJoinItem"))) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER_INSTRUCTIONS).withJoinPath(List.of("orderJoinCurrentOrderInstructions")).withLabel("Current Order Instructions")) .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE)) .withField(new QFieldMetaData("billToPersonId", QFieldType.INTEGER).withBackendName("bill_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) .withField(new QFieldMetaData("shipToPersonId", QFieldType.INTEGER).withBackendName("ship_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java index 18890603..0847c58d 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java @@ -23,26 +23,42 @@ package com.kingsrook.qqq.backend.module.rdbms.reporting; import java.io.ByteArrayOutputStream; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.RenderSavedReportMetaDataProducer; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest; +import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -196,6 +212,164 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest } + /******************************************************************************* + ** + *******************************************************************************/ + private List runSavedReportForCSV(SavedReport newSavedReport) throws Exception + { + newSavedReport.setLabel("Test Report"); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, null); + + QRecord savedReport = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(newSavedReport)).getRecords().get(0); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName(RenderSavedReportMetaDataProducer.NAME); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + input.setCallback(QProcessCallbackFactory.forRecord(savedReport)); + input.addValue("reportFormat", ReportFormatPossibleValueEnum.CSV.getPossibleValueId()); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(input); + + return (FileUtils.readLines(new File(runProcessOutput.getValueString("serverFilePath")), StandardCharsets.UTF_8)); + } + + + + /******************************************************************************* + ** in here, by potentially ambiguous, we mean where there are possible joins + ** between the order and orderInstructions tables. + *******************************************************************************/ + @Test + void testSavedReportWithPotentiallyAmbiguousExposedJoinSelections() throws Exception + { + List lines = runSavedReportForCSV(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(""" + {"columns":[ + {"name": "id"}, + {"name": "storeId"}, + {"name": "orderInstructions.instructions"} + ]}""") + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()))); + + assertEquals(""" + "Id","Store","Instructions" + """.trim(), lines.get(0)); + assertEquals(""" + "1","Q-Mart","order 1 v2" + """.trim(), lines.get(1)); + } + + + /******************************************************************************* + ** in here, by potentially ambiguous, we mean where there are possible joins + ** between the order and orderInstructions tables. + *******************************************************************************/ + @Test + void testSavedReportWithPotentiallyAmbiguousExposedJoinSelectedAndOrdered() throws Exception + { + List lines = runSavedReportForCSV(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(""" + {"columns":[ + {"name": "id"}, + {"name": "storeId"}, + {"name": "orderInstructions.instructions"} + ]}""") + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter() + .withOrderBy(new QFilterOrderBy("orderInstructions.id", false)) + ))); + + assertEquals(""" + "Id","Store","Instructions" + """.trim(), lines.get(0)); + assertEquals(""" + "8","QDepot","order 8 v1" + """.trim(), lines.get(1)); + } + + + /******************************************************************************* + ** in here, by potentially ambiguous, we mean where there are possible joins + ** between the order and orderInstructions tables. + *******************************************************************************/ + @Test + void testSavedReportWithPotentiallyAmbiguousExposedJoinCriteria() throws Exception + { + List lines = runSavedReportForCSV(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(""" + {"columns":[ + {"name": "id"}, + {"name": "storeId"} + ]}""") + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter() + .withCriteria(new QFilterCriteria("orderInstructions.instructions", QCriteriaOperator.CONTAINS, "v3")) + ))); + + assertEquals(""" + "Id","Store" + """.trim(), lines.get(0)); + assertEquals(""" + "2","Q-Mart" + """.trim(), lines.get(1)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSavedReportWithExposedJoinMultipleTablesAwaySelected() throws Exception + { + List lines = runSavedReportForCSV(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(""" + {"columns":[ + {"name": "id"}, + {"name": "storeId"}, + {"name": "item.description"} + ]}""") + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()))); + + assertEquals(""" + "Id","Store","Description" + """.trim(), lines.get(0)); + assertEquals(""" + "1","Q-Mart","Q-Mart Item 1" + """.trim(), lines.get(1)); + } + + // todo - similar to above, but w/o selecting, only filtering + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSavedReportWithExposedJoinMultipleTablesAwayAsCriteria() throws Exception + { + List lines = runSavedReportForCSV(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(""" + {"columns":[ + {"name": "id"}, + {"name": "storeId"} + ]}""") + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter() + .withCriteria(new QFilterCriteria("item.description", QCriteriaOperator.CONTAINS, "Item 7")) + ))); + + assertEquals(""" + "Id","Store" + """.trim(), lines.get(0)); + assertEquals(""" + "6","QDepot" + """.trim(), lines.get(1)); + } + + /******************************************************************************* **