mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 05:01:07 +00:00
Merged dev into feature/join-enhancements
This commit is contained in:
@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.AbstractRDBMSAction;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSAggregateAction;
|
||||
@ -55,7 +56,10 @@ public class RDBMSBackendModule implements QBackendModuleInterface
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(RDBMSBackendModule.class);
|
||||
|
||||
|
||||
static
|
||||
{
|
||||
QBackendModuleDispatcher.registerBackendModule(new RDBMSBackendModule());
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Method where a backend module must be able to provide its type (name).
|
||||
|
||||
@ -857,6 +857,17 @@ public abstract class AbstractRDBMSAction
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Make it easy (e.g., for tests) to turn on logging of SQL
|
||||
*******************************************************************************/
|
||||
public static void setLogSQL(boolean on, boolean doReformat, String loggerOrSystemOut)
|
||||
{
|
||||
setLogSQL(on);
|
||||
setLogSQLOutput(loggerOrSystemOut);
|
||||
setLogSQLReformat(doReformat);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Make it easy (e.g., for tests) to turn on logging of SQL
|
||||
*******************************************************************************/
|
||||
|
||||
@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
@ -46,6 +47,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager;
|
||||
@ -61,6 +63,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
public class TestUtils
|
||||
{
|
||||
public static final String DEFAULT_BACKEND_NAME = "default";
|
||||
public static final String MEMORY_BACKEND_NAME = "memory";
|
||||
|
||||
public static final String TABLE_NAME_PERSON = "personTable";
|
||||
public static final String TABLE_NAME_PERSONAL_ID_CARD = "personalIdCard";
|
||||
@ -107,6 +110,7 @@ public class TestUtils
|
||||
{
|
||||
QInstance qInstance = new QInstance();
|
||||
qInstance.addBackend(defineBackend());
|
||||
qInstance.addBackend(defineMemoryBackend());
|
||||
qInstance.addTable(defineTablePerson());
|
||||
qInstance.addPossibleValueSource(definePvsPerson());
|
||||
qInstance.addTable(defineTablePersonalIdCard());
|
||||
@ -118,6 +122,18 @@ public class TestUtils
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Define the in-memory backend used in standard tests
|
||||
*******************************************************************************/
|
||||
public static QBackendMetaData defineMemoryBackend()
|
||||
{
|
||||
return new QBackendMetaData()
|
||||
.withName(MEMORY_BACKEND_NAME)
|
||||
.withBackendType(MemoryBackendModule.class);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Define the authentication used in standard tests - using 'mock' type.
|
||||
**
|
||||
@ -243,7 +259,7 @@ public class TestUtils
|
||||
.withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId"))
|
||||
.withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_ORDER_LINE).withJoinName("orderJoinOrderLine"))
|
||||
.withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ITEM).withJoinPath(List.of("orderJoinOrderLine", "orderLineJoinItem")))
|
||||
.withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER_INSTRUCTIONS).withJoinPath(List.of("orderJoinCurrentOrderInstructions")))
|
||||
.withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER_INSTRUCTIONS).withJoinPath(List.of("orderJoinCurrentOrderInstructions")).withLabel("Current Order Instructions"))
|
||||
.withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE))
|
||||
.withField(new QFieldMetaData("billToPersonId", QFieldType.INTEGER).withBackendName("bill_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON))
|
||||
.withField(new QFieldMetaData("shipToPersonId", QFieldType.INTEGER).withBackendName("ship_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON))
|
||||
|
||||
@ -23,25 +23,52 @@ package com.kingsrook.qqq.backend.module.rdbms.reporting;
|
||||
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableFunction;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType;
|
||||
import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns;
|
||||
import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport;
|
||||
import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryStorageAction;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.RenderSavedReportMetaDataProducer;
|
||||
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.LocalMacDevUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.TestUtils;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
@ -196,6 +223,220 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private RunProcessOutput runSavedReport(SavedReport savedReport, ReportFormatPossibleValueEnum reportFormat) throws Exception
|
||||
{
|
||||
savedReport.setLabel("Test Report");
|
||||
QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
|
||||
|
||||
if(QContext.getQInstance().getTable(SavedReport.TABLE_NAME) == null)
|
||||
{
|
||||
new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null);
|
||||
}
|
||||
|
||||
QRecord savedReportRecord = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(savedReport)).getRecords().get(0);
|
||||
|
||||
RunProcessInput input = new RunProcessInput();
|
||||
input.setProcessName(RenderSavedReportMetaDataProducer.NAME);
|
||||
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
|
||||
input.setCallback(QProcessCallbackFactory.forRecord(savedReportRecord));
|
||||
input.addValue("reportFormat", reportFormat);
|
||||
RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
|
||||
return (runProcessOutput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private List<String> runSavedReportForCSV(SavedReport savedReport) throws Exception
|
||||
{
|
||||
RunProcessOutput runProcessOutput = runSavedReport(savedReport, ReportFormatPossibleValueEnum.CSV);
|
||||
|
||||
String storageTableName = runProcessOutput.getValueString("storageTableName");
|
||||
String storageReference = runProcessOutput.getValueString("storageReference");
|
||||
InputStream inputStream = new MemoryStorageAction().getInputStream(new StorageInput(storageTableName).withReference(storageReference));
|
||||
|
||||
return (IOUtils.readLines(inputStream, StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** in here, by potentially ambiguous, we mean where there are possible joins
|
||||
** between the order and orderInstructions tables.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSavedReportWithPotentiallyAmbiguousExposedJoinSelections() throws Exception
|
||||
{
|
||||
List<String> lines = runSavedReportForCSV(new SavedReport()
|
||||
.withTableName(TestUtils.TABLE_NAME_ORDER)
|
||||
.withColumnsJson(JsonUtils.toJson(new ReportColumns()
|
||||
.withColumn("id")
|
||||
.withColumn("storeId")
|
||||
.withColumn("orderInstructions.instructions")))
|
||||
.withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())));
|
||||
|
||||
assertEquals("""
|
||||
"Id","Store","Order Instructions: Instructions"
|
||||
""".trim(), lines.get(0));
|
||||
assertEquals("""
|
||||
"1","Q-Mart","order 1 v2"
|
||||
""".trim(), lines.get(1));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** in here, by potentially ambiguous, we mean where there are possible joins
|
||||
** between the order and orderInstructions tables.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSavedReportWithPotentiallyAmbiguousExposedJoinSelectedAndOrdered() throws Exception
|
||||
{
|
||||
List<String> lines = runSavedReportForCSV(new SavedReport()
|
||||
.withTableName(TestUtils.TABLE_NAME_ORDER)
|
||||
.withColumnsJson(JsonUtils.toJson(new ReportColumns()
|
||||
.withColumn("id")
|
||||
.withColumn("storeId")
|
||||
.withColumn("orderInstructions.instructions")))
|
||||
.withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()
|
||||
.withOrderBy(new QFilterOrderBy("orderInstructions.id", false))
|
||||
)));
|
||||
|
||||
assertEquals("""
|
||||
"Id","Store","Order Instructions: Instructions"
|
||||
""".trim(), lines.get(0));
|
||||
assertEquals("""
|
||||
"8","QDepot","order 8 v1"
|
||||
""".trim(), lines.get(1));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** in here, by potentially ambiguous, we mean where there are possible joins
|
||||
** between the order and orderInstructions tables.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSavedReportWithPotentiallyAmbiguousExposedJoinCriteria() throws Exception
|
||||
{
|
||||
List<String> lines = runSavedReportForCSV(new SavedReport()
|
||||
.withTableName(TestUtils.TABLE_NAME_ORDER)
|
||||
.withColumnsJson(JsonUtils.toJson(new ReportColumns()
|
||||
.withColumn("id")
|
||||
.withColumn("storeId")))
|
||||
.withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()
|
||||
.withCriteria(new QFilterCriteria("orderInstructions.instructions", QCriteriaOperator.CONTAINS, "v3"))
|
||||
)));
|
||||
|
||||
assertEquals("""
|
||||
"Id","Store"
|
||||
""".trim(), lines.get(0));
|
||||
assertEquals("""
|
||||
"2","Q-Mart"
|
||||
""".trim(), lines.get(1));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSavedReportWithExposedJoinMultipleTablesAwaySelected() throws Exception
|
||||
{
|
||||
List<String> lines = runSavedReportForCSV(new SavedReport()
|
||||
.withTableName(TestUtils.TABLE_NAME_ORDER)
|
||||
.withColumnsJson(JsonUtils.toJson(new ReportColumns()
|
||||
.withColumn("id")
|
||||
.withColumn("storeId")
|
||||
.withColumn("item.description")))
|
||||
.withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())));
|
||||
|
||||
assertEquals("""
|
||||
"Id","Store","Item: Description"
|
||||
""".trim(), lines.get(0));
|
||||
assertEquals("""
|
||||
"1","Q-Mart","Q-Mart Item 1"
|
||||
""".trim(), lines.get(1));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSavedReportWithExposedJoinMultipleTablesAwayAsCriteria() throws Exception
|
||||
{
|
||||
List<String> lines = runSavedReportForCSV(new SavedReport()
|
||||
.withTableName(TestUtils.TABLE_NAME_ORDER)
|
||||
.withColumnsJson(JsonUtils.toJson(new ReportColumns()
|
||||
.withColumn("id")
|
||||
.withColumn("storeId")))
|
||||
.withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()
|
||||
.withCriteria(new QFilterCriteria("item.description", QCriteriaOperator.CONTAINS, "Item 7"))
|
||||
)));
|
||||
|
||||
assertEquals("""
|
||||
"Id","Store"
|
||||
""".trim(), lines.get(0));
|
||||
assertEquals("""
|
||||
"6","QDepot"
|
||||
""".trim(), lines.get(1));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSavedReportWithPivotsFromJoinTable() throws Exception
|
||||
{
|
||||
SavedReport savedReport = new SavedReport()
|
||||
.withTableName(TestUtils.TABLE_NAME_ORDER)
|
||||
.withColumnsJson(JsonUtils.toJson(new ReportColumns()
|
||||
.withColumn("id")
|
||||
.withColumn("item.storeId")
|
||||
.withColumn("item.description")))
|
||||
.withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()))
|
||||
.withPivotTableJson(JsonUtils.toJson(new PivotTableDefinition()
|
||||
.withRow(new PivotTableGroupBy().withFieldName("item.storeId"))
|
||||
.withValue(new PivotTableValue().withFieldName("item.description").withFunction(PivotTableFunction.COUNT))));
|
||||
|
||||
//////////////////////////////////////////////
|
||||
// make sure we can render xlsx w/o a crash //
|
||||
//////////////////////////////////////////////
|
||||
RunProcessOutput runProcessOutput = runSavedReport(savedReport, ReportFormatPossibleValueEnum.XLSX);
|
||||
String storageTableName = runProcessOutput.getValueString("storageTableName");
|
||||
String storageReference = runProcessOutput.getValueString("storageReference");
|
||||
InputStream inputStream = new MemoryStorageAction().getInputStream(new StorageInput(storageTableName).withReference(storageReference));
|
||||
|
||||
String path = "/tmp/pivot.xlsx";
|
||||
inputStream.transferTo(new FileOutputStream(path));
|
||||
// LocalMacDevUtils.mayOpenFiles = true;
|
||||
LocalMacDevUtils.openFile(path);
|
||||
|
||||
///////////////////////////////////////////////////////
|
||||
// render as csv too - and assert about those values //
|
||||
///////////////////////////////////////////////////////
|
||||
List<String> csv = runSavedReportForCSV(savedReport);
|
||||
System.out.println(StringUtils.join("\n", csv));
|
||||
assertEquals("""
|
||||
"Store","Count Of Item: Description\"""", csv.get(0));
|
||||
assertEquals("""
|
||||
"Q-Mart","4\"""", csv.get(1));
|
||||
assertEquals("""
|
||||
"Totals","11\"""", csv.get(4));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -204,9 +445,12 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest
|
||||
ReportInput reportInput = new ReportInput();
|
||||
QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
|
||||
reportInput.setReportName(TEST_REPORT);
|
||||
reportInput.setReportFormat(ReportFormat.CSV);
|
||||
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
reportInput.setReportOutputStream(outputStream);
|
||||
|
||||
reportInput.setReportDestination(new ReportDestination()
|
||||
.withReportFormat(ReportFormat.CSV)
|
||||
.withReportOutputStream(outputStream));
|
||||
|
||||
new GenerateReportAction().execute(reportInput);
|
||||
return (outputStream.toString());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user