diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java index 8fe2046d..e5669615 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java @@ -34,19 +34,23 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction; import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.reporting.customizers.DataSourceQueryInputCustomizer; import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportViewCustomizer; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QFormulaException; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; @@ -54,6 +58,9 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -79,6 +86,8 @@ import com.kingsrook.qqq.backend.core.utils.aggregates.InstantAggregates; import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates; import com.kingsrook.qqq.backend.core.utils.aggregates.LocalDateAggregates; import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates; +import com.kingsrook.qqq.backend.core.utils.aggregates.StringAggregates; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -93,7 +102,7 @@ import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates; ** - summaries (like pivot tables, but called summary to avoid confusion with "native" pivot tables), ** - native pivot tables (not initially supported, due to lack of support in fastexcel...). *******************************************************************************/ -public class GenerateReportAction +public class GenerateReportAction extends AbstractQActionFunction { private static final QLogger LOG = QLogger.getLogger(GenerateReportAction.class); @@ -117,13 +126,16 @@ public class GenerateReportAction private List dataSources; private List views; + private Map countByDataSource = new HashMap<>(); + /******************************************************************************* ** *******************************************************************************/ - public void execute(ReportInput reportInput) throws QException + public ReportOutput execute(ReportInput reportInput) throws QException { + ReportOutput reportOutput = new ReportOutput(); QReportMetaData report = getReportMetaData(reportInput); this.views = report.getViews(); @@ -203,21 +215,27 @@ public class GenerateReportAction //////////////////////////////////////////////////////////////////////////////////// // start the table-view (e.g., open this tab in xlsx) and then run the query-loop // //////////////////////////////////////////////////////////////////////////////////// - startTableView(reportInput, dataSource, dataSourceTableView); + startTableView(reportInput, dataSource, dataSourceTableView, reportFormat); gatherData(reportInput, dataSource, dataSourceTableView, dataSourceSummaryViews, dataSourceVariantViews); } } } - //////////////////////////////////////// - // add pivot sheets // - // todo - but, only for Excel, right? // - //////////////////////////////////////// + ////////////////////// + // add pivot sheets // + ////////////////////// for(QReportView view : views) { if(view.getType().equals(ReportType.PIVOT)) { - startTableView(reportInput, null, view); + if(reportFormat.getSupportsNativePivotTables()) + { + startTableView(reportInput, null, view, reportFormat); + } + else + { + LOG.warn("Request to render a report with a PIVOT type view, for a format that does not support native pivot tables", logPair("reportFormat", reportFormat)); + } ////////////////////////////////////////////////////////////////////////// // there's no data to add to a pivot table, so nothing else to do here. // @@ -227,6 +245,8 @@ public class GenerateReportAction outputSummaries(reportInput); + reportOutput.setTotalRecordCount(countByDataSource.values().stream().mapToInt(Integer::intValue).sum()); + reportStreamer.finish(); try @@ -237,6 +257,8 @@ public class GenerateReportAction { throw (new QReportingException("Error completing report", e)); } + + return (reportOutput); } @@ -264,7 +286,7 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView) throws QException + private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView, ReportFormat reportFormat) throws QException { QMetaDataVariableInterpreter variableInterpreter = new QMetaDataVariableInterpreter(); variableInterpreter.addValueMap("input", reportInput.getInputValues()); @@ -281,6 +303,8 @@ public class GenerateReportAction { joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), dataSource.getQueryFilter()); } + + countDataSourceRecords(reportInput, dataSource, reportFormat); } List fields = new ArrayList<>(); @@ -318,7 +342,36 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List summaryViews, List variantViews) throws QException + private void countDataSourceRecords(ReportInput reportInput, QReportDataSource dataSource, ReportFormat reportFormat) throws QException + { + QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone(); + setInputValuesInQueryFilter(reportInput, queryFilter); + + CountInput countInput = new CountInput(); + countInput.setTableName(dataSource.getSourceTable()); + countInput.setFilter(queryFilter); + countInput.setQueryJoins(dataSource.getQueryJoins()); + CountOutput countOutput = new CountAction().execute(countInput); + + if(countOutput.getCount() != null) + { + countByDataSource.put(dataSource.getName(), countOutput.getCount()); + + if(reportFormat.getMaxRows() != null && countOutput.getCount() > reportFormat.getMaxRows()) + { + throw (new QUserFacingException("The requested report would include more rows (" + + String.format("%,d", countOutput.getCount()) + ") than the maximum allowed (" + + String.format("%,d", reportFormat.getMaxRows()) + ") for the selected file format (" + reportFormat + ").")); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private Integer gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List summaryViews, List variantViews) throws QException { //////////////////////////////////////////////////////////////////////////////////////// // check if this view has a transform step - if so, set it up now and run its pre-run // @@ -345,6 +398,9 @@ public class GenerateReportAction RunBackendStepInput finalTransformStepInput = transformStepInput; RunBackendStepOutput finalTransformStepOutput = transformStepOutput; + String tableLabel = QContext.getQInstance().getTable(dataSource.getSourceTable()).getLabel(); + AtomicInteger consumedCount = new AtomicInteger(0); + ///////////////////////////////////////////////////////////////// // run a record pipe loop, over the query for this data source // ///////////////////////////////////////////////////////////////// @@ -405,6 +461,17 @@ public class GenerateReportAction records = finalTransformStepOutput.getRecords(); } + Integer total = countByDataSource.get(dataSource.getName()); + if(total != null) + { + reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records", consumedCount.get() + 1, total); + } + else + { + reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records (" + String.format("%,d", consumedCount.get() + 1) + ")"); + } + consumedCount.getAndAdd(records.size()); + return (consumeRecords(reportInput, dataSource, records, tableView, summaryViews, variantViews)); }); @@ -415,6 +482,8 @@ public class GenerateReportAction { transformStep.postRun(new BackendStepPostRunInput(transformStepInput), new BackendStepPostRunOutput(transformStepOutput)); } + + return consumedCount.get(); } @@ -422,7 +491,7 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private Set setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext) + private Set setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext) throws QException { Set fieldsToTranslatePossibleValues = new HashSet<>(); @@ -440,15 +509,16 @@ public class GenerateReportAction } } - for(String summaryField : CollectionUtils.nonNullList(view.getSummaryFields())) + for(String summaryFieldName : CollectionUtils.nonNullList(view.getSummaryFields())) { /////////////////////////////////////////////////////////////////////////////// // all pivotFields that are possible value sources are implicitly translated // /////////////////////////////////////////////////////////////////////////////// - QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable()); - if(table.getField(summaryField).getPossibleValueSourceName() != null) + QTableMetaData mainTable = QContext.getQInstance().getTable(dataSource.getSourceTable()); + FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(mainTable, summaryFieldName); + if(fieldAndJoinTable.field().getPossibleValueSourceName() != null) { - fieldsToTranslatePossibleValues.add(summaryField); + fieldsToTranslatePossibleValues.add(summaryFieldName); } } } @@ -458,6 +528,32 @@ public class GenerateReportAction + /******************************************************************************* + ** + *******************************************************************************/ + private FieldAndJoinTable getFieldAndJoinTable(QTableMetaData mainTable, String fieldName) throws QException + { + if(fieldName.indexOf('.') > -1) + { + String joinTableName = fieldName.replaceAll("\\..*", ""); + String joinFieldName = fieldName.replaceAll(".*\\.", ""); + + QTableMetaData joinTable = QContext.getQInstance().getTable(joinTableName); + if(joinTable == null) + { + throw (new QException("Unrecognized join table name: " + joinTableName)); + } + + return new FieldAndJoinTable(joinTable.getField(joinFieldName), joinTable); + } + else + { + return new FieldAndJoinTable(mainTable.getField(fieldName), mainTable); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -550,24 +646,25 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List records, Map>>> aggregatesMap) + private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List records, Map>>> aggregatesMap) throws QException { Map>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>()); for(QRecord record : records) { SummaryKey key = new SummaryKey(); - for(String summaryField : view.getSummaryFields()) + for(String summaryFieldName : view.getSummaryFields()) { - Serializable summaryValue = record.getValue(summaryField); - if(table.getField(summaryField).getPossibleValueSourceName() != null) + FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName); + Serializable summaryValue = record.getValue(summaryFieldName); + if(fieldAndJoinTable.field().getPossibleValueSourceName() != null) { ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // so, this is kinda a thing - where we implicitly use possible-value labels (e.g., display values) for pivot fields... // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - summaryValue = record.getDisplayValue(summaryField); + summaryValue = record.getDisplayValue(summaryFieldName); } - key.add(summaryField, summaryValue); + key.add(summaryFieldName, summaryValue); if(view.getIncludeSummarySubTotals() && key.getKeys().size() < view.getSummaryFields().size()) { @@ -588,7 +685,7 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map>> viewAggregates, SummaryKey key) + private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map>> viewAggregates, SummaryKey key) throws QException { Map> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>()); addRecordToAggregatesMap(table, record, keyAggregates); @@ -599,44 +696,57 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map> aggregatesMap) + private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map> aggregatesMap) throws QException { - for(QFieldMetaData field : table.getFields().values()) + ////////////////////////////////////////////////////////////////////////////////////// + // todo - an optimization could be, to only compute aggregates that we'll need... // + // Only if we measure and see this to be slow - it may be, lots of BigDecimal math? // + ////////////////////////////////////////////////////////////////////////////////////// + for(String fieldName : record.getValues().keySet()) { + FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, fieldName); + QFieldMetaData field = fieldAndJoinTable.field(); if(StringUtils.hasContent(field.getPossibleValueSourceName())) { - continue; + @SuppressWarnings("unchecked") + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates()); + fieldAggregates.add(record.getDisplayValue(fieldName)); } - - if(field.getType().equals(QFieldType.INTEGER)) + else if(field.getType().equals(QFieldType.INTEGER)) { @SuppressWarnings("unchecked") - AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new IntegerAggregates()); - fieldAggregates.add(record.getValueInteger(field.getName())); + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new IntegerAggregates()); + fieldAggregates.add(record.getValueInteger(fieldName)); } else if(field.getType().equals(QFieldType.LONG)) { @SuppressWarnings("unchecked") - AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new LongAggregates()); - fieldAggregates.add(record.getValueLong(field.getName())); + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LongAggregates()); + fieldAggregates.add(record.getValueLong(fieldName)); } else if(field.getType().equals(QFieldType.DECIMAL)) { @SuppressWarnings("unchecked") - AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new BigDecimalAggregates()); - fieldAggregates.add(record.getValueBigDecimal(field.getName())); + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new BigDecimalAggregates()); + fieldAggregates.add(record.getValueBigDecimal(fieldName)); } else if(field.getType().equals(QFieldType.DATE_TIME)) { @SuppressWarnings("unchecked") - AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new InstantAggregates()); - fieldAggregates.add(record.getValueInstant(field.getName())); + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new InstantAggregates()); + fieldAggregates.add(record.getValueInstant(fieldName)); } else if(field.getType().equals(QFieldType.DATE)) { @SuppressWarnings("unchecked") - AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new LocalDateAggregates()); - fieldAggregates.add(record.getValueLocalDate(field.getName())); + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LocalDateAggregates()); + fieldAggregates.add(record.getValueLocalDate(fieldName)); + } + if(field.getType().isStringLike()) + { + @SuppressWarnings("unchecked") + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates()); + fieldAggregates.add(record.getValueString(fieldName)); } } } @@ -646,7 +756,7 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void outputSummaries(ReportInput reportInput) throws QReportingException, QFormulaException + private void outputSummaries(ReportInput reportInput) throws QException { List reportViews = views.stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList(); for(QReportView view : reportViews) @@ -719,13 +829,13 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private List getFields(QTableMetaData table, QReportView view) + private List getFields(QTableMetaData table, QReportView view) throws QException { List fields = new ArrayList<>(); - for(String summaryField : view.getSummaryFields()) + for(String summaryFieldName : view.getSummaryFields()) { - QFieldMetaData field = table.getField(summaryField); - fields.add(new QFieldMetaData(summaryField, field.getType()).withLabel(field.getLabel())); // todo do we need the type? if so need table as input here + FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName); + fields.add(new QFieldMetaData(summaryFieldName, fieldAndJoinTable.field().getType()).withLabel(fieldAndJoinTable.field().getLabel())); // todo do we need the type? if so need table as input here } for(QReportField column : view.getColumns()) { @@ -982,4 +1092,10 @@ public class GenerateReportAction { } + + + /******************************************************************************* + ** + *******************************************************************************/ + private record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable) {} } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java index 139051a7..0b798fe8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java @@ -30,6 +30,9 @@ import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; @@ -39,7 +42,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.JsonUtils; -import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; /******************************************************************************* @@ -64,6 +67,9 @@ public class JsonExportStreamer implements ExportStreamerInterface private byte[] indent = new byte[0]; private String indentString = ""; + private Pattern colonLetterPattern = Pattern.compile(":([A-Z]+)($|[A-Z][a-z])"); + private Memoization fieldLabelMemoization = new Memoization<>(); + /******************************************************************************* @@ -232,8 +238,7 @@ public class JsonExportStreamer implements ExportStreamerInterface Map mapForJson = new LinkedHashMap<>(); for(QFieldMetaData field : fields) { - String labelForJson = StringUtils.lcFirst(field.getLabel().replace(" ", "")); - mapForJson.put(labelForJson, qRecord.getValue(field.getName())); + mapForJson.put(getLabelForJson(field), qRecord.getValue(field.getName())); } String json = prettyPrint ? JsonUtils.toPrettyJson(mapForJson) : JsonUtils.toJson(mapForJson); @@ -261,6 +266,73 @@ public class JsonExportStreamer implements ExportStreamerInterface + /******************************************************************************* + ** + *******************************************************************************/ + String getLabelForJson(QFieldMetaData field) + { + ////////////////////////////////////////////////////////////////////////// + // memoize, to avoid running these regex/replacements millions of times // + ////////////////////////////////////////////////////////////////////////// + Optional result = fieldLabelMemoization.getResult(field.getName(), fieldName -> + { + String labelForJson = field.getLabel().replace(" ", ""); + + ///////////////////////////////////////////////////////////////////////////// + // now fix up any field-name-parts after the table: portion of a join name // + // lineItem:SKU to become lineItem:sku // + // parcel:SLAStatus to become parcel:slaStatus // + // order:Client to become order:client // + ///////////////////////////////////////////////////////////////////////////// + Matcher allCaps = Pattern.compile("^[A-Z]+$").matcher(labelForJson); + Matcher startsAllCapsThenNextWordMatcher = Pattern.compile("([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson); + Matcher startsOneCapMatcher = Pattern.compile("([A-Z])(.*)").matcher(labelForJson); + + if(allCaps.matches()) + { + labelForJson = allCaps.replaceAll(m -> m.group().toLowerCase()); + } + else if(startsAllCapsThenNextWordMatcher.matches()) + { + labelForJson = startsAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2)); + } + else if(startsOneCapMatcher.matches()) + { + labelForJson = startsOneCapMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2)); + } + + ///////////////////////////////////////////////////////////////////////////// + // now fix up any field-name-parts after the table: portion of a join name // + // lineItem:SKU to become lineItem:sku // + // parcel:SLAStatus to become parcel:slaStatus // + // order:Client to become order:client // + ///////////////////////////////////////////////////////////////////////////// + Matcher colonThenAllCapsThenEndMatcher = Pattern.compile("(.*:)([A-Z]+)$").matcher(labelForJson); + Matcher colonThenAllCapsThenNextWordMatcher = Pattern.compile("(.*:)([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson); + Matcher colonThenOneCapMatcher = Pattern.compile("(.*:)([A-Z])(.*)").matcher(labelForJson); + + if(colonThenAllCapsThenEndMatcher.matches()) + { + labelForJson = colonThenAllCapsThenEndMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase()); + } + else if(colonThenAllCapsThenNextWordMatcher.matches()) + { + labelForJson = colonThenAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3)); + } + else if(colonThenOneCapMatcher.matches()) + { + labelForJson = colonThenOneCapMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3)); + } + + System.out.println("Label: " + labelForJson); + return (labelForJson); + }); + + return result.orElse(field.getName()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java index 01c11efd..ead1fe41 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java @@ -70,6 +70,7 @@ import org.apache.poi.ss.usermodel.DataConsolidateFunction; import org.apache.poi.ss.usermodel.DateUtil; import org.apache.poi.ss.util.AreaReference; import org.apache.poi.ss.util.CellReference; +import org.apache.poi.ss.util.WorkbookUtil; import org.apache.poi.xssf.usermodel.XSSFCell; import org.apache.poi.xssf.usermodel.XSSFCellStyle; import org.apache.poi.xssf.usermodel.XSSFPivotTable; @@ -151,6 +152,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter for(QReportView view : views) { String label = Objects.requireNonNullElse(view.getLabel(), "Sheet " + sheetCounter); + label = WorkbookUtil.createSafeSheetName(label); ///////////////////////////////////////////////////////////////////////////////////////////// // track the actually-used sheet labels (needed for referencing in pivot table generation) // @@ -637,18 +639,6 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter { throw (new QReportingException("Error adding totals row", e)); } - - /* todo - CellStyle totalsStyle = workbook.createCellStyle(); - Font font = workbook.createFont(); - font.setBold(true); - totalsStyle.setFont(font); - totalsStyle.setBorderTop(BorderStyle.THIN); - totalsStyle.setBorderTop(BorderStyle.THIN); - totalsStyle.setBorderBottom(BorderStyle.DOUBLE); - - row.cellIterator().forEachRemaining(cell -> cell.setCellStyle(totalsStyle)); - */ } @@ -666,9 +656,10 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter ////////////////////////////////////////////// closeLastSheetIfOpen(); - ///////////////////////////// - // close the output stream // - ///////////////////////////// + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // note - leave the zipOutputStream open. It is a wrapper around the OutputStream we were given by the caller, // + // so it is their responsibility to close that stream (which implicitly closes the zip, it appears) // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// zipOutputStream.close(); } catch(Exception e) @@ -784,7 +775,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter """, - labelViewsByName.get(dataView.getName()), + StreamedPoiSheetWriter.cleanseValue(labelViewsByName.get(dataView.getName())), CellReference.convertNumToColString(dataView.getColumns().size() - 1), rowsPerView.get(dataView.getName()), dataView.getColumns().size(), diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedPoiSheetWriter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedPoiSheetWriter.java index 3b343de9..5579634a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedPoiSheetWriter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedPoiSheetWriter.java @@ -121,7 +121,7 @@ public class StreamedPoiSheetWriter /******************************************************************************* ** *******************************************************************************/ - private String cleanseValue(String value) + public static String cleanseValue(String value) { // todo - profile... if(xmlSpecialChars.matcher(value).find()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java index 369cbc6a..be901e94 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java @@ -32,7 +32,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; /******************************************************************************* - ** Input for an Export action + ** Input for a Report action *******************************************************************************/ public class ReportInput extends AbstractTableActionInput { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportOutput.java new file mode 100644 index 00000000..7c51ad8e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportOutput.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting; + + +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; + + +/******************************************************************************* + ** Output for a Report action + *******************************************************************************/ +public class ReportOutput extends AbstractActionOutput implements Serializable +{ + private Integer totalRecordCount; + + + + /******************************************************************************* + ** Getter for totalRecordCount + *******************************************************************************/ + public Integer getTotalRecordCount() + { + return (this.totalRecordCount); + } + + + + /******************************************************************************* + ** Setter for totalRecordCount + *******************************************************************************/ + public void setTotalRecordCount(Integer totalRecordCount) + { + this.totalRecordCount = totalRecordCount; + } + + + + /******************************************************************************* + ** Fluent setter for totalRecordCount + *******************************************************************************/ + public ReportOutput withTotalRecordCount(Integer totalRecordCount) + { + this.totalRecordCount = totalRecordCount; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java index 1f6e6f43..d83c3784 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java @@ -77,6 +77,7 @@ public class SavedReportToReportMetaDataAdapter QInstance qInstance = QContext.getQInstance(); QReportMetaData reportMetaData = new QReportMetaData(); + reportMetaData.setName("savedReport:" + savedReport.getId()); reportMetaData.setLabel(savedReport.getLabel()); ///////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/StringAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/StringAggregates.java new file mode 100644 index 00000000..33e306f1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/StringAggregates.java @@ -0,0 +1,121 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.aggregates; + + +/******************************************************************************* + ** String version of data aggregator + *******************************************************************************/ +public class StringAggregates implements AggregatesInterface +{ + private int count = 0; + + private String min; + private String max; + + + + /******************************************************************************* + ** Add a new value to this aggregate set + *******************************************************************************/ + public void add(String input) + { + if(input == null) + { + return; + } + + count++; + + if(min == null || input.compareTo(min) < 0) + { + min = input; + } + + if(max == null || input.compareTo(max) > 0) + { + max = input; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int getCount() + { + return (count); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getSum() + { + ////////////////////////////////////// + // sum of string doesn't make sense // + ////////////////////////////////////// + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getMin() + { + return (min); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getMax() + { + return (max); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getAverage() + { + /////////////////////////////////////// + // average string doesn't make sense // + /////////////////////////////////////// + return (null); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamerTest.java new file mode 100644 index 00000000..126272bc --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamerTest.java @@ -0,0 +1,54 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for JsonExportStreamer + *******************************************************************************/ +class JsonExportStreamerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + Function runOne = label -> new JsonExportStreamer().getLabelForJson(new QFieldMetaData("test", QFieldType.STRING).withLabel(label)); + assertEquals("sku", runOne.apply("SKU")); + assertEquals("clientName", runOne.apply("Client Name")); + assertEquals("slaStatus", runOne.apply("SLA Status")); + assertEquals("lineItem:sku", runOne.apply("Line Item: SKU")); + assertEquals("parcel:slaStatus", runOne.apply("Parcel: SLA Status")); + assertEquals("order:client", runOne.apply("Order: Client")); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStream.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStream.java index 2a3cd416..fa35bee3 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStream.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStream.java @@ -36,6 +36,8 @@ import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.UploadPartResult; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -45,16 +47,20 @@ import com.amazonaws.services.s3.model.UploadPartResult; *******************************************************************************/ public class S3UploadOutputStream extends OutputStream { + private static final QLogger LOG = QLogger.getLogger(S3UploadOutputStream.class); + private final AmazonS3 amazonS3; private final String bucketName; private final String key; - private byte[] buffer = new byte[5 * 1024 * 1024]; - private int offset = 0; + private byte[] buffer = new byte[5 * 1024 * 1024]; + private int offset = 0; private InitiateMultipartUploadResult initiateMultipartUploadResult = null; private List uploadPartResultList = null; + private boolean isClosed = false; + /******************************************************************************* @@ -96,10 +102,12 @@ public class S3UploadOutputStream extends OutputStream ////////////////////////////////////////// if(initiateMultipartUploadResult == null) { + LOG.info("Initiating a multipart upload", logPair("key", key)); initiateMultipartUploadResult = amazonS3.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucketName, key)); uploadPartResultList = new ArrayList<>(); } + LOG.info("Uploading a part", logPair("key", key), logPair("partNumber", uploadPartResultList.size() + 1)); UploadPartRequest uploadPartRequest = new UploadPartRequest() .withUploadId(initiateMultipartUploadResult.getUploadId()) .withPartNumber(uploadPartResultList.size() + 1) @@ -130,7 +138,6 @@ public class S3UploadOutputStream extends OutputStream while(bytesToWrite > buffer.length - offset) { int size = buffer.length - offset; - // System.out.println("A:copy " + size + " bytes from source[" + off + "] to dest[" + offset + "]"); System.arraycopy(b, off, buffer, offset, size); offset = buffer.length; uploadIfNeeded(); @@ -139,7 +146,6 @@ public class S3UploadOutputStream extends OutputStream } int size = len - off; - // System.out.println("B:copy " + size + " bytes from source[" + off + "] to dest[" + offset + "]"); System.arraycopy(b, off, buffer, offset, size); offset += size; uploadIfNeeded(); @@ -153,13 +159,20 @@ public class S3UploadOutputStream extends OutputStream @Override public void close() throws IOException { + if(isClosed) + { + LOG.debug("Redundant call to close an already-closed S3UploadOutputStream. Returning with noop.", logPair("key", key)); + return; + } + if(initiateMultipartUploadResult != null) { - if (offset > 0) + if(offset > 0) { ////////////////////////////////////////////////// // if there's a final part to upload, do it now // ////////////////////////////////////////////////// + LOG.info("Uploading a part", logPair("key", key), logPair("isFinalPart", true), logPair("partNumber", uploadPartResultList.size() + 1)); UploadPartRequest uploadPartRequest = new UploadPartRequest() .withUploadId(initiateMultipartUploadResult.getUploadId()) .withPartNumber(uploadPartResultList.size() + 1) @@ -179,10 +192,13 @@ public class S3UploadOutputStream extends OutputStream } else { + LOG.info("Putting object (non-multipart)", logPair("key", key), logPair("length", offset)); ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setContentLength(offset); PutObjectResult putObjectResult = amazonS3.putObject(bucketName, key, new ByteArrayInputStream(buffer, 0, offset), objectMetadata); } + + isClosed = true; } } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index 68c99d28..057128f3 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -63,16 +63,16 @@ public class TestUtils public static final String BACKEND_NAME_S3 = "s3"; public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; public static final String BACKEND_NAME_MOCK = "mock"; - public static final String BACKEND_NAME_MEMORY = "memory"; + public static final String BACKEND_NAME_MEMORY = "memory"; public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json"; public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv"; public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob"; - public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive"; + public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive"; public static final String TABLE_NAME_PERSON_S3 = "person-s3"; public static final String TABLE_NAME_BLOB_S3 = "s3-blob"; public static final String TABLE_NAME_PERSON_MOCK = "person-mock"; - public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix"; + public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix"; public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed"; public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter"; diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java index f9d545a9..9e3be3ed 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.module.rdbms.reporting; import java.io.ByteArrayOutputStream; +import java.io.FileOutputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.List; @@ -38,6 +39,10 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableFunction; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; @@ -59,6 +64,8 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryStorageAction; import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.RenderSavedReportMetaDataProducer; import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.LocalMacDevUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest; import org.apache.commons.io.IOUtils; @@ -215,23 +222,39 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest } + /******************************************************************************* ** *******************************************************************************/ - private List runSavedReportForCSV(SavedReport newSavedReport) throws Exception + private RunProcessOutput runSavedReport(SavedReport savedReport, ReportFormatPossibleValueEnum reportFormat) throws Exception { - newSavedReport.setLabel("Test Report"); + savedReport.setLabel("Test Report"); QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); - QRecord savedReport = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(newSavedReport)).getRecords().get(0); + if(QContext.getQInstance().getTable(SavedReport.TABLE_NAME) == null) + { + new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + } + + QRecord savedReportRecord = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(savedReport)).getRecords().get(0); RunProcessInput input = new RunProcessInput(); input.setProcessName(RenderSavedReportMetaDataProducer.NAME); input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); - input.setCallback(QProcessCallbackFactory.forRecord(savedReport)); - input.addValue("reportFormat", ReportFormatPossibleValueEnum.CSV.getPossibleValueId()); + input.setCallback(QProcessCallbackFactory.forRecord(savedReportRecord)); + input.addValue("reportFormat", reportFormat); RunProcessOutput runProcessOutput = new RunProcessAction().execute(input); + return (runProcessOutput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List runSavedReportForCSV(SavedReport savedReport) throws Exception + { + RunProcessOutput runProcessOutput = runSavedReport(savedReport, ReportFormatPossibleValueEnum.CSV); String storageTableName = runProcessOutput.getValueString("storageTableName"); String storageReference = runProcessOutput.getValueString("storageReference"); @@ -293,6 +316,7 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest } + /******************************************************************************* ** in here, by potentially ambiguous, we mean where there are possible joins ** between the order and orderInstructions tables. @@ -341,7 +365,6 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest """.trim(), lines.get(1)); } - // todo - similar to above, but w/o selecting, only filtering /******************************************************************************* @@ -369,6 +392,51 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSavedReportWithPivotsFromJoinTable() throws Exception + { + SavedReport savedReport = new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("item.storeId") + .withColumn("item.description"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withPivotTableJson(JsonUtils.toJson(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("item.storeId")) + .withValue(new PivotTableValue().withFieldName("item.description").withFunction(PivotTableFunction.COUNT)))); + + ////////////////////////////////////////////// + // make sure we can render xlsx w/o a crash // + ////////////////////////////////////////////// + RunProcessOutput runProcessOutput = runSavedReport(savedReport, ReportFormatPossibleValueEnum.XLSX); + String storageTableName = runProcessOutput.getValueString("storageTableName"); + String storageReference = runProcessOutput.getValueString("storageReference"); + InputStream inputStream = new MemoryStorageAction().getInputStream(new StorageInput(storageTableName).withReference(storageReference)); + + String path = "/tmp/pivot.xlsx"; + inputStream.transferTo(new FileOutputStream(path)); + // LocalMacDevUtils.mayOpenFiles = true; + LocalMacDevUtils.openFile(path); + + /////////////////////////////////////////////////////// + // render as csv too - and assert about those values // + /////////////////////////////////////////////////////// + List csv = runSavedReportForCSV(savedReport); + System.out.println(StringUtils.join("\n", csv)); + assertEquals(""" + "Store","Count Of Item: Description\"""", csv.get(0)); + assertEquals(""" + "Q-Mart","4\"""", csv.get(1)); + assertEquals(""" + "Totals","11\"""", csv.get(4)); + } + + + /******************************************************************************* ** *******************************************************************************/