CE-881 - Add queryJoins when rendering savedReports - tested in RDBMS module

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java

@@ -23,8 +23,10 @@ package com.kingsrook.qqq.backend.core.processes.implementations.savedreports;
 
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -32,14 +34,17 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@@ -63,6 +68,8 @@ public class SavedReportToReportMetaDataAdapter
    {
       try
       {
+         QInstance qInstance = QContext.getQInstance();
+
          QReportMetaData reportMetaData = new QReportMetaData();
          reportMetaData.setLabel(savedReport.getLabel());
 
@@ -73,15 +80,11 @@ public class SavedReportToReportMetaDataAdapter
          reportMetaData.setDataSources(List.of(dataSource));
          dataSource.setName("main");
 
-         QTableMetaData table = QContext.getQInstance().getTable(savedReport.getTableName());
+         QTableMetaData table = qInstance.getTable(savedReport.getTableName());
          dataSource.setSourceTable(savedReport.getTableName());
 
          dataSource.setQueryFilter(JsonUtils.toObject(savedReport.getQueryFilterJson(), QQueryFilter.class));
 
-         // todo!!! oh my.
-         List<QueryJoin> queryJoins = null;
-         dataSource.setQueryJoins(queryJoins);
-
          //////////////////////////
          // set up the main view //
          //////////////////////////
@@ -103,10 +106,13 @@ public class SavedReportToReportMetaDataAdapter
          ///////////////////////////////////////////////////////////////////////////////////////////////////////////
          // columns in the saved-report look like a JSON object, w/ a key "columns", which is an array of objects //
          ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+         Set<String> neededJoinTables = new HashSet<>();
+
+         List<QReportField> reportColumns = new ArrayList<>();
+         view.setColumns(reportColumns);
+
          Map<String, Object>       columnsObject = JsonUtils.toObject(savedReport.getColumnsJson(), new TypeReference<>() {});
          List<Map<String, Object>> columns       = (List<Map<String, Object>>) columnsObject.get("columns");
-         List<QReportField>        reportColumns = new ArrayList<>();
-
          for(Map<String, Object> column : columns)
          {
             if(column.containsKey("isVisible") && !"true".equals(ValueUtils.getValueAsString(column.get("isVisible"))))
@@ -114,11 +120,28 @@ public class SavedReportToReportMetaDataAdapter
                continue;
             }
 
-            QFieldMetaData field = null;
+            QFieldMetaData field;
-            String fieldName = ValueUtils.getValueAsString(column.get("name"));
+            String         fieldName = ValueUtils.getValueAsString(column.get("name"));
             if(fieldName.contains("."))
             {
-               // todo - join!
+               String joinTableName = fieldName.replaceAll("\\..*", "");
+               String joinFieldName = fieldName.replaceAll(".*\\.", "");
+
+               QTableMetaData joinTable = qInstance.getTable(joinTableName);
+               if(joinTable == null)
+               {
+                  LOG.warn("Saved Report has an unrecognized join table name", logPair("savedReportId", savedReport.getId()), logPair("joinTable", joinTable), logPair("fieldName", fieldName));
+                  continue;
+               }
+
+               neededJoinTables.add(joinTableName);
+
+               field = joinTable.getFields().get(joinFieldName);
+               if(field == null)
+               {
+                  LOG.warn("Saved Report has an unrecognized join field name", logPair("savedReportId", savedReport.getId()), logPair("fieldName", fieldName));
+                  continue;
+               }
             }
             else
             {
@@ -148,7 +171,40 @@ public class SavedReportToReportMetaDataAdapter
             }
          }
 
-         view.setColumns(reportColumns);
+         ///////////////////////////////////////////////////////////////////////////////////////////
+         // set up joins, if we need any                                                          //
+         // note - test coverage here is provided by RDBMS module's GenerateReportActionRDBMSTest //
+         ///////////////////////////////////////////////////////////////////////////////////////////
+         if(!neededJoinTables.isEmpty())
+         {
+            List<QueryJoin> queryJoins = new ArrayList<>();
+            dataSource.setQueryJoins(queryJoins);
+
+            for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
+            {
+               if(neededJoinTables.contains(exposedJoin.getJoinTable()))
+               {
+                  QueryJoin queryJoin = new QueryJoin(exposedJoin.getJoinTable())
+                     .withSelect(true)
+                     .withType(QueryJoin.Type.LEFT)
+                     .withBaseTableOrAlias(null)
+                     .withAlias(null);
+
+                  if(exposedJoin.getJoinPath().size() == 1)
+                  {
+                     // this is similar logic that QFMD has
+
+                     //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                     // todo - what about a join with a longer path?  it would be nice to pass such joinNames through there too, //
+                     // but what, that would actually be multiple queryJoins?  needs a fair amount of thought.                   //
+                     //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                     queryJoin.setJoinMetaData(qInstance.getJoin(exposedJoin.getJoinPath().get(0)));
+                  }
+
+                  queryJoins.add(queryJoin);
+               }
+            }
+         }
 
          ///////////////////////////////////////////////
          // if it's a pivot report, add that view too //

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java

@@ -954,6 +954,17 @@ public abstract class AbstractRDBMSAction
 
 
 
+   /*******************************************************************************
+    ** Make it easy (e.g., for tests) to turn on logging of SQL
+    *******************************************************************************/
+   public static void setLogSQL(boolean on, boolean doReformat, String loggerOrSystemOut)
+   {
+      setLogSQL(on);
+      setLogSQLOutput(loggerOrSystemOut);
+      setLogSQLReformat(doReformat);
+   }
+
+
    /*******************************************************************************
     ** Make it easy (e.g., for tests) to turn on logging of SQL
     *******************************************************************************/

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java

@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@@ -46,6 +47,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
 import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest;
 import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
 import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager;
@@ -61,6 +63,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
 public class TestUtils
 {
    public static final String DEFAULT_BACKEND_NAME = "default";
+   public static final String MEMORY_BACKEND_NAME  = "memory";
 
    public static final String TABLE_NAME_PERSON              = "personTable";
    public static final String TABLE_NAME_PERSONAL_ID_CARD    = "personalIdCard";
@@ -107,6 +110,7 @@ public class TestUtils
    {
       QInstance qInstance = new QInstance();
       qInstance.addBackend(defineBackend());
+      qInstance.addBackend(defineMemoryBackend());
       qInstance.addTable(defineTablePerson());
       qInstance.addPossibleValueSource(definePvsPerson());
       qInstance.addTable(defineTablePersonalIdCard());
@@ -118,6 +122,18 @@ public class TestUtils
 
 
 
+   /*******************************************************************************
+    ** Define the in-memory backend used in standard tests
+    *******************************************************************************/
+   public static QBackendMetaData defineMemoryBackend()
+   {
+      return new QBackendMetaData()
+         .withName(MEMORY_BACKEND_NAME)
+         .withBackendType(MemoryBackendModule.class);
+   }
+
+
+
    /*******************************************************************************
     ** Define the authentication used in standard tests - using 'mock' type.
     **
@@ -243,6 +259,7 @@ public class TestUtils
          .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId"))
          .withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_ORDER_LINE).withJoinName("orderJoinOrderLine"))
          .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ITEM).withJoinPath(List.of("orderJoinOrderLine", "orderLineJoinItem")))
+         .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER_INSTRUCTIONS).withJoinPath(List.of("orderJoinCurrentOrderInstructions")).withLabel("Current Order Instructions"))
          .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE))
          .withField(new QFieldMetaData("billToPersonId", QFieldType.INTEGER).withBackendName("bill_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON))
          .withField(new QFieldMetaData("shipToPersonId", QFieldType.INTEGER).withBackendName("ship_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON))

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java

@@ -23,26 +23,42 @@ package com.kingsrook.qqq.backend.module.rdbms.reporting;
 
 
 import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
 import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType;
+import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport;
+import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.RenderSavedReportMetaDataProducer;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.module.rdbms.TestUtils;
 import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest;
+import org.apache.commons.io.FileUtils;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -196,6 +212,164 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest
    }
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<String> runSavedReportForCSV(SavedReport newSavedReport) throws Exception
+   {
+      newSavedReport.setLabel("Test Report");
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
+      new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, null);
+
+      QRecord savedReport = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(newSavedReport)).getRecords().get(0);
+
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName(RenderSavedReportMetaDataProducer.NAME);
+      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+      input.setCallback(QProcessCallbackFactory.forRecord(savedReport));
+      input.addValue("reportFormat", ReportFormatPossibleValueEnum.CSV.getPossibleValueId());
+      RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
+
+      return (FileUtils.readLines(new File(runProcessOutput.getValueString("serverFilePath")), StandardCharsets.UTF_8));
+   }
+
+
+
+   /*******************************************************************************
+    ** in here, by potentially ambiguous, we mean where there are possible joins
+    ** between the order and orderInstructions tables.
+    *******************************************************************************/
+   @Test
+   void testSavedReportWithPotentiallyAmbiguousExposedJoinSelections() throws Exception
+   {
+      List<String> lines = runSavedReportForCSV(new SavedReport()
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withColumnsJson("""
+            {"columns":[
+               {"name": "id"},
+               {"name": "storeId"},
+               {"name": "orderInstructions.instructions"}
+            ]}""")
+         .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())));
+
+      assertEquals("""
+         "Id","Store","Instructions"
+         """.trim(), lines.get(0));
+      assertEquals("""
+         "1","Q-Mart","order 1 v2"
+         """.trim(), lines.get(1));
+   }
+
+
+   /*******************************************************************************
+    ** in here, by potentially ambiguous, we mean where there are possible joins
+    ** between the order and orderInstructions tables.
+    *******************************************************************************/
+   @Test
+   void testSavedReportWithPotentiallyAmbiguousExposedJoinSelectedAndOrdered() throws Exception
+   {
+      List<String> lines = runSavedReportForCSV(new SavedReport()
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withColumnsJson("""
+            {"columns":[
+               {"name": "id"},
+               {"name": "storeId"},
+               {"name": "orderInstructions.instructions"}
+            ]}""")
+         .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()
+            .withOrderBy(new QFilterOrderBy("orderInstructions.id", false))
+         )));
+
+      assertEquals("""
+         "Id","Store","Instructions"
+         """.trim(), lines.get(0));
+      assertEquals("""
+         "8","QDepot","order 8 v1"
+         """.trim(), lines.get(1));
+   }
+
+
+   /*******************************************************************************
+    ** in here, by potentially ambiguous, we mean where there are possible joins
+    ** between the order and orderInstructions tables.
+    *******************************************************************************/
+   @Test
+   void testSavedReportWithPotentiallyAmbiguousExposedJoinCriteria() throws Exception
+   {
+      List<String> lines = runSavedReportForCSV(new SavedReport()
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withColumnsJson("""
+            {"columns":[
+               {"name": "id"},
+               {"name": "storeId"}
+            ]}""")
+         .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()
+            .withCriteria(new QFilterCriteria("orderInstructions.instructions", QCriteriaOperator.CONTAINS, "v3"))
+         )));
+
+      assertEquals("""
+         "Id","Store"
+         """.trim(), lines.get(0));
+      assertEquals("""
+         "2","Q-Mart"
+         """.trim(), lines.get(1));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSavedReportWithExposedJoinMultipleTablesAwaySelected() throws Exception
+   {
+      List<String> lines = runSavedReportForCSV(new SavedReport()
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withColumnsJson("""
+            {"columns":[
+               {"name": "id"},
+               {"name": "storeId"},
+               {"name": "item.description"}
+            ]}""")
+         .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())));
+
+      assertEquals("""
+         "Id","Store","Description"
+         """.trim(), lines.get(0));
+      assertEquals("""
+         "1","Q-Mart","Q-Mart Item 1"
+         """.trim(), lines.get(1));
+   }
+
+   // todo - similar to above, but w/o selecting, only filtering
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSavedReportWithExposedJoinMultipleTablesAwayAsCriteria() throws Exception
+   {
+      List<String> lines = runSavedReportForCSV(new SavedReport()
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withColumnsJson("""
+            {"columns":[
+               {"name": "id"},
+               {"name": "storeId"}
+            ]}""")
+         .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()
+            .withCriteria(new QFilterCriteria("item.description", QCriteriaOperator.CONTAINS, "Item 7"))
+         )));
+
+      assertEquals("""
+         "Id","Store"
+         """.trim(), lines.get(0));
+      assertEquals("""
+         "6","QDepot"
+         """.trim(), lines.get(1));
+   }
+
+
 
    /*******************************************************************************
     **