QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

handle ExposedJoins in exports

Browse Source
This commit is contained in:
darin.kelkhoff
2023-04-26 10:19:12 -05:00
parent f7f001d430
commit 8094c29ec7
4 changed files with 178 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java
View File

@ -23,8 +23,12 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager;
@ -32,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
@ -41,10 +46,13 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@ -95,15 +103,25 @@ public class ExportAction
///////////////////////////////////
if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames()))
{
QTableMetaData table = exportInput.getTable();
List<String> badFieldNames = new ArrayList<>();
QTableMetaData table = exportInput.getTable();
Map<String, QTableMetaData> joinTableMap = getJoinTableMap(table);
List<String> badFieldNames = new ArrayList<>();
for(String fieldName : exportInput.getFieldNames())
{
try
{
table.getField(fieldName);
if(fieldName.contains("."))
{
String[] parts = fieldName.split("\\.", 2);
joinTableMap.get(parts[0]).getField(parts[1]);
}
else
{
table.getField(fieldName);
}
}
catch(IllegalArgumentException iae)
catch(Exception e)
{
badFieldNames.add(fieldName);
}
@ -128,6 +146,21 @@ public class ExportAction
/*******************************************************************************
**
*******************************************************************************/
private static Map<String, QTableMetaData> getJoinTableMap(QTableMetaData table)
{
Map<String, QTableMetaData> joinTableMap = new HashMap<>();
for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
{
joinTableMap.put(exposedJoin.getJoinTable(), QContext.getQInstance().getTable(exposedJoin.getJoinTable()));
}
return joinTableMap;
}
/*******************************************************************************
** Run the report.
*******************************************************************************/
@ -151,7 +184,33 @@ public class ExportAction
QueryInput queryInput = new QueryInput();
queryInput.setTableName(exportInput.getTableName());
queryInput.setFilter(exportInput.getQueryFilter());
queryInput.setLimit(exportInput.getLimit());
List<QueryJoin> queryJoins = new ArrayList<>();
Set<String> addedJoinNames = new HashSet<>();
if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames()))
{
for(String fieldName : exportInput.getFieldNames())
{
if(fieldName.contains("."))
{
String[] parts = fieldName.split("\\.", 2);
String joinTableName = parts[0];
if(!addedJoinNames.contains(joinTableName))
{
queryJoins.add(new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true));
addedJoinNames.add(joinTableName);
}
}
}
}
queryInput.setQueryJoins(queryJoins);
if(queryInput.getFilter() == null)
{
queryInput.setFilter(new QQueryFilter());
}
queryInput.getFilter().setLimit(exportInput.getLimit());
queryInput.setShouldTranslatePossibleValues(true);
/////////////////////////////////////////////////////////////////
@ -298,11 +357,29 @@ public class ExportAction
*******************************************************************************/
private List<QFieldMetaData> getFields(ExportInput exportInput)
{
QTableMetaData table = exportInput.getTable();
Map<String, QTableMetaData> joinTableMap = getJoinTableMap(table);
List<QFieldMetaData> fieldList;
QTableMetaData table = exportInput.getTable();
if(exportInput.getFieldNames() != null)
{
fieldList = exportInput.getFieldNames().stream().map(table::getField).toList();
fieldList = new ArrayList<>();
for(String fieldName : exportInput.getFieldNames())
{
if(fieldName.contains("."))
{
String[] parts = fieldName.split("\\.", 2);
QTableMetaData joinTable = joinTableMap.get(parts[0]);
QFieldMetaData field = joinTable.getField(parts[1]).clone();
field.setName(fieldName);
field.setLabel(joinTable.getLabel() + ": " + field.getLabel());
fieldList.add(field);
}
else
{
fieldList.add(table.getField(fieldName));
}
}
}
else
{

28
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java
View File

@ -38,13 +38,16 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@ -181,7 +184,7 @@ public class MemoryRecordStore
}
BackendQueryFilterUtils.sortRecordList(input.getFilter(), records);
records = BackendQueryFilterUtils.applySkipAndLimit(input, records);
records = BackendQueryFilterUtils.applySkipAndLimit(input.getFilter(), records);
return (records);
}
@ -191,8 +194,11 @@ public class MemoryRecordStore
/*******************************************************************************
**
*******************************************************************************/
private Collection<QRecord> buildJoinCrossProduct(QueryInput input)
private Collection<QRecord> buildJoinCrossProduct(QueryInput input) throws QException
{
QInstance qInstance = QContext.getQInstance();
JoinsContext joinsContext = new JoinsContext(qInstance, input.getTableName(), input.getQueryJoins(), input.getFilter());
List<QRecord> crossProduct = new ArrayList<>();
QTableMetaData leftTable = input.getTable();
for(QRecord record : getTableData(leftTable).values())
@ -204,16 +210,26 @@ public class MemoryRecordStore
for(QueryJoin queryJoin : input.getQueryJoins())
{
QTableMetaData nextTable = QContext.getQInstance().getTable(queryJoin.getJoinTable());
QTableMetaData nextTable = qInstance.getTable(queryJoin.getJoinTable());
Collection<QRecord> nextTableRecords = getTableData(nextTable).values();
QJoinMetaData joinMetaData = Objects.requireNonNullElseGet(queryJoin.getJoinMetaData(), () ->
{
QJoinMetaData found = joinsContext.findJoinMetaData(qInstance, input.getTableName(), queryJoin.getJoinTable());
if(found == null)
{
throw (new RuntimeException("Could not find a join between tables [" + input.getTableName() + "][" + queryJoin.getJoinTable() + "]"));
}
return (found);
});
List<QRecord> nextLevelProduct = new ArrayList<>();
for(QRecord productRecord : crossProduct)
{
boolean matchFound = false;
for(QRecord nextTableRecord : nextTableRecords)
{
if(joinMatches(productRecord, nextTableRecord, queryJoin))
if(joinMatches(productRecord, nextTableRecord, queryJoin, joinMetaData))
{
QRecord joinRecord = new QRecord(productRecord);
addRecordToProduct(joinRecord, nextTableRecord, queryJoin.getJoinTableOrItsAlias());
@ -239,9 +255,9 @@ public class MemoryRecordStore
/*******************************************************************************
**
*******************************************************************************/
private boolean joinMatches(QRecord productRecord, QRecord nextTableRecord, QueryJoin queryJoin)
private boolean joinMatches(QRecord productRecord, QRecord nextTableRecord, QueryJoin queryJoin, QJoinMetaData joinMetaData)
{
for(JoinOn joinOn : queryJoin.getJoinMetaData().getJoinOns())
for(JoinOn joinOn : joinMetaData.getJoinOns())
{
Serializable leftValue = productRecord.getValues().containsKey(queryJoin.getBaseTableOrAlias() + "." + joinOn.getLeftField())
? productRecord.getValue(queryJoin.getBaseTableOrAlias() + "." + joinOn.getLeftField())

69
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java
View File

@ -22,10 +22,12 @@
package com.kingsrook.qqq.backend.core.actions.reporting;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.BaseTest;
@ -36,16 +38,21 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.io.FileUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ -117,6 +124,68 @@ class ExportActionTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testJoins() throws QException, IOException
{
QInstance qInstance = QContext.getQInstance();
QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true);
TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_ORDER), List.of(
new QRecord().withValue("id", 1).withValue("orderNo", "ORD1").withValue("storeId", 1),
new QRecord().withValue("id", 2).withValue("orderNo", "ORD2").withValue("storeId", 1)
));
TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_LINE_ITEM), List.of(
new QRecord().withValue("id", 1).withValue("orderId", 1).withValue("sku", "A").withValue("quantity", 10),
new QRecord().withValue("id", 2).withValue("orderId", 1).withValue("sku", "B").withValue("quantity", 15),
new QRecord().withValue("id", 3).withValue("orderId", 2).withValue("sku", "A").withValue("quantity", 20)
));
ExportInput exportInput = new ExportInput();
exportInput.setTableName(TestUtils.TABLE_NAME_ORDER);
exportInput.setReportFormat(ReportFormat.CSV);
ByteArrayOutputStream reportOutputStream = new ByteArrayOutputStream();
exportInput.setReportOutputStream(reportOutputStream);
exportInput.setQueryFilter(new QQueryFilter());
exportInput.setFieldNames(List.of("id", "orderNo", "storeId", "orderLine.id", "orderLine.sku", "orderLine.quantity"));
// exportInput.setFieldNames(List.of("id", "orderNo", "storeId"));
new ExportAction().execute(exportInput);
String csv = reportOutputStream.toString(StandardCharsets.UTF_8);
CSVParser parse = CSVParser.parse(csv, CSVFormat.DEFAULT.withFirstRecordAsHeader());
Iterator<CSVRecord> csvRecordIterator = parse.iterator();
assertFalse(parse.getHeaderMap().isEmpty());
assertTrue(parse.getHeaderMap().containsKey("Id"));
assertTrue(parse.getHeaderMap().containsKey("Order Line: Id"));
assertTrue(parse.getHeaderMap().containsKey("Order Line: SKU"));
CSVRecord csvRecord = csvRecordIterator.next();
assertEquals("1", csvRecord.get("Id"));
assertEquals("1", csvRecord.get("Order Line: Id"));
assertEquals("A", csvRecord.get("Order Line: SKU"));
assertEquals("10", csvRecord.get("Order Line: Quantity"));
csvRecord = csvRecordIterator.next();
assertEquals("1", csvRecord.get("Id"));
assertEquals("2", csvRecord.get("Order Line: Id"));
assertEquals("B", csvRecord.get("Order Line: SKU"));
assertEquals("15", csvRecord.get("Order Line: Quantity"));
csvRecord = csvRecordIterator.next();
assertEquals("2", csvRecord.get("Id"));
assertEquals("3", csvRecord.get("Order Line: Id"));
assertEquals("A", csvRecord.get("Order Line: SKU"));
assertEquals("20", csvRecord.get("Order Line: Quantity"));
assertFalse(csvRecordIterator.hasNext());
}
/*******************************************************************************
**
*******************************************************************************/

4
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -92,6 +92,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTracking;
@ -547,6 +548,7 @@ public class TestUtils
.withFieldName("storeId"))
.withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_LINE_ITEM).withJoinName("orderLineItem"))
.withAssociation(new Association().withName("extrinsics").withAssociatedTableName(TABLE_NAME_ORDER_EXTRINSIC).withJoinName("orderOrderExtrinsic"))
.withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_LINE_ITEM))
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
@ -581,7 +583,7 @@ public class TestUtils
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("orderId", QFieldType.INTEGER))
.withField(new QFieldMetaData("lineNumber", QFieldType.STRING))
.withField(new QFieldMetaData("sku", QFieldType.STRING))
.withField(new QFieldMetaData("sku", QFieldType.STRING).withLabel("SKU"))
.withField(new QFieldMetaData("quantity", QFieldType.INTEGER));
}