diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java index ce0a0920..2ca95347 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java @@ -23,8 +23,12 @@ package com.kingsrook.qqq.backend.core.actions.reporting; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager; @@ -32,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; @@ -41,10 +46,13 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -95,15 +103,25 @@ public class ExportAction /////////////////////////////////// if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames())) { - QTableMetaData table = exportInput.getTable(); - List badFieldNames = new ArrayList<>(); + QTableMetaData table = exportInput.getTable(); + Map joinTableMap = getJoinTableMap(table); + + List badFieldNames = new ArrayList<>(); for(String fieldName : exportInput.getFieldNames()) { try { - table.getField(fieldName); + if(fieldName.contains(".")) + { + String[] parts = fieldName.split("\\.", 2); + joinTableMap.get(parts[0]).getField(parts[1]); + } + else + { + table.getField(fieldName); + } } - catch(IllegalArgumentException iae) + catch(Exception e) { badFieldNames.add(fieldName); } @@ -128,6 +146,21 @@ public class ExportAction + /******************************************************************************* + ** + *******************************************************************************/ + private static Map getJoinTableMap(QTableMetaData table) + { + Map joinTableMap = new HashMap<>(); + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + joinTableMap.put(exposedJoin.getJoinTable(), QContext.getQInstance().getTable(exposedJoin.getJoinTable())); + } + return joinTableMap; + } + + + /******************************************************************************* ** Run the report. *******************************************************************************/ @@ -151,7 +184,33 @@ public class ExportAction QueryInput queryInput = new QueryInput(); queryInput.setTableName(exportInput.getTableName()); queryInput.setFilter(exportInput.getQueryFilter()); - queryInput.setLimit(exportInput.getLimit()); + + List queryJoins = new ArrayList<>(); + Set addedJoinNames = new HashSet<>(); + if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames())) + { + for(String fieldName : exportInput.getFieldNames()) + { + if(fieldName.contains(".")) + { + String[] parts = fieldName.split("\\.", 2); + String joinTableName = parts[0]; + if(!addedJoinNames.contains(joinTableName)) + { + queryJoins.add(new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true)); + addedJoinNames.add(joinTableName); + } + } + } + } + + queryInput.setQueryJoins(queryJoins); + + if(queryInput.getFilter() == null) + { + queryInput.setFilter(new QQueryFilter()); + } + queryInput.getFilter().setLimit(exportInput.getLimit()); queryInput.setShouldTranslatePossibleValues(true); ///////////////////////////////////////////////////////////////// @@ -298,11 +357,29 @@ public class ExportAction *******************************************************************************/ private List getFields(ExportInput exportInput) { + QTableMetaData table = exportInput.getTable(); + Map joinTableMap = getJoinTableMap(table); + List fieldList; - QTableMetaData table = exportInput.getTable(); if(exportInput.getFieldNames() != null) { - fieldList = exportInput.getFieldNames().stream().map(table::getField).toList(); + fieldList = new ArrayList<>(); + for(String fieldName : exportInput.getFieldNames()) + { + if(fieldName.contains(".")) + { + String[] parts = fieldName.split("\\.", 2); + QTableMetaData joinTable = joinTableMap.get(parts[0]); + QFieldMetaData field = joinTable.getField(parts[1]).clone(); + field.setName(fieldName); + field.setLabel(joinTable.getLabel() + ": " + field.getLabel()); + fieldList.add(field); + } + else + { + fieldList.add(table.getField(fieldName)); + } + } } else { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java index 6cc00b8f..b25cac78 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java @@ -38,13 +38,16 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -181,7 +184,7 @@ public class MemoryRecordStore } BackendQueryFilterUtils.sortRecordList(input.getFilter(), records); - records = BackendQueryFilterUtils.applySkipAndLimit(input, records); + records = BackendQueryFilterUtils.applySkipAndLimit(input.getFilter(), records); return (records); } @@ -191,8 +194,11 @@ public class MemoryRecordStore /******************************************************************************* ** *******************************************************************************/ - private Collection buildJoinCrossProduct(QueryInput input) + private Collection buildJoinCrossProduct(QueryInput input) throws QException { + QInstance qInstance = QContext.getQInstance(); + JoinsContext joinsContext = new JoinsContext(qInstance, input.getTableName(), input.getQueryJoins(), input.getFilter()); + List crossProduct = new ArrayList<>(); QTableMetaData leftTable = input.getTable(); for(QRecord record : getTableData(leftTable).values()) @@ -204,16 +210,26 @@ public class MemoryRecordStore for(QueryJoin queryJoin : input.getQueryJoins()) { - QTableMetaData nextTable = QContext.getQInstance().getTable(queryJoin.getJoinTable()); + QTableMetaData nextTable = qInstance.getTable(queryJoin.getJoinTable()); Collection nextTableRecords = getTableData(nextTable).values(); + QJoinMetaData joinMetaData = Objects.requireNonNullElseGet(queryJoin.getJoinMetaData(), () -> + { + QJoinMetaData found = joinsContext.findJoinMetaData(qInstance, input.getTableName(), queryJoin.getJoinTable()); + if(found == null) + { + throw (new RuntimeException("Could not find a join between tables [" + input.getTableName() + "][" + queryJoin.getJoinTable() + "]")); + } + return (found); + }); + List nextLevelProduct = new ArrayList<>(); for(QRecord productRecord : crossProduct) { boolean matchFound = false; for(QRecord nextTableRecord : nextTableRecords) { - if(joinMatches(productRecord, nextTableRecord, queryJoin)) + if(joinMatches(productRecord, nextTableRecord, queryJoin, joinMetaData)) { QRecord joinRecord = new QRecord(productRecord); addRecordToProduct(joinRecord, nextTableRecord, queryJoin.getJoinTableOrItsAlias()); @@ -239,9 +255,9 @@ public class MemoryRecordStore /******************************************************************************* ** *******************************************************************************/ - private boolean joinMatches(QRecord productRecord, QRecord nextTableRecord, QueryJoin queryJoin) + private boolean joinMatches(QRecord productRecord, QRecord nextTableRecord, QueryJoin queryJoin, QJoinMetaData joinMetaData) { - for(JoinOn joinOn : queryJoin.getJoinMetaData().getJoinOns()) + for(JoinOn joinOn : joinMetaData.getJoinOns()) { Serializable leftValue = productRecord.getValues().containsKey(queryJoin.getBaseTableOrAlias() + "." + joinOn.getLeftField()) ? productRecord.getValue(queryJoin.getBaseTableOrAlias() + "." + joinOn.getLeftField()) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java index a4fccc5c..9ff5f7de 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java @@ -22,10 +22,12 @@ package com.kingsrook.qqq.backend.core.actions.reporting; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.BaseTest; @@ -36,16 +38,21 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; import org.apache.commons.io.FileUtils; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -117,6 +124,68 @@ class ExportActionTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testJoins() throws QException, IOException + { + QInstance qInstance = QContext.getQInstance(); + QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true); + + TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_ORDER), List.of( + new QRecord().withValue("id", 1).withValue("orderNo", "ORD1").withValue("storeId", 1), + new QRecord().withValue("id", 2).withValue("orderNo", "ORD2").withValue("storeId", 1) + )); + + TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_LINE_ITEM), List.of( + new QRecord().withValue("id", 1).withValue("orderId", 1).withValue("sku", "A").withValue("quantity", 10), + new QRecord().withValue("id", 2).withValue("orderId", 1).withValue("sku", "B").withValue("quantity", 15), + new QRecord().withValue("id", 3).withValue("orderId", 2).withValue("sku", "A").withValue("quantity", 20) + )); + + ExportInput exportInput = new ExportInput(); + exportInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + exportInput.setReportFormat(ReportFormat.CSV); + ByteArrayOutputStream reportOutputStream = new ByteArrayOutputStream(); + exportInput.setReportOutputStream(reportOutputStream); + exportInput.setQueryFilter(new QQueryFilter()); + exportInput.setFieldNames(List.of("id", "orderNo", "storeId", "orderLine.id", "orderLine.sku", "orderLine.quantity")); + // exportInput.setFieldNames(List.of("id", "orderNo", "storeId")); + new ExportAction().execute(exportInput); + + String csv = reportOutputStream.toString(StandardCharsets.UTF_8); + CSVParser parse = CSVParser.parse(csv, CSVFormat.DEFAULT.withFirstRecordAsHeader()); + Iterator csvRecordIterator = parse.iterator(); + assertFalse(parse.getHeaderMap().isEmpty()); + assertTrue(parse.getHeaderMap().containsKey("Id")); + assertTrue(parse.getHeaderMap().containsKey("Order Line: Id")); + assertTrue(parse.getHeaderMap().containsKey("Order Line: SKU")); + + CSVRecord csvRecord = csvRecordIterator.next(); + assertEquals("1", csvRecord.get("Id")); + assertEquals("1", csvRecord.get("Order Line: Id")); + assertEquals("A", csvRecord.get("Order Line: SKU")); + assertEquals("10", csvRecord.get("Order Line: Quantity")); + + csvRecord = csvRecordIterator.next(); + assertEquals("1", csvRecord.get("Id")); + assertEquals("2", csvRecord.get("Order Line: Id")); + assertEquals("B", csvRecord.get("Order Line: SKU")); + assertEquals("15", csvRecord.get("Order Line: Quantity")); + + csvRecord = csvRecordIterator.next(); + assertEquals("2", csvRecord.get("Id")); + assertEquals("3", csvRecord.get("Order Line: Id")); + assertEquals("A", csvRecord.get("Order Line: SKU")); + assertEquals("20", csvRecord.get("Order Line: Quantity")); + + assertFalse(csvRecordIterator.hasNext()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 70b74a32..1fad436f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -92,6 +92,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTracking; @@ -547,6 +548,7 @@ public class TestUtils .withFieldName("storeId")) .withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_LINE_ITEM).withJoinName("orderLineItem")) .withAssociation(new Association().withName("extrinsics").withAssociatedTableName(TABLE_NAME_ORDER_EXTRINSIC).withJoinName("orderOrderExtrinsic")) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_LINE_ITEM)) .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) @@ -581,7 +583,7 @@ public class TestUtils .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("orderId", QFieldType.INTEGER)) .withField(new QFieldMetaData("lineNumber", QFieldType.STRING)) - .withField(new QFieldMetaData("sku", QFieldType.STRING)) + .withField(new QFieldMetaData("sku", QFieldType.STRING).withLabel("SKU")) .withField(new QFieldMetaData("quantity", QFieldType.INTEGER)); }