CE-881 - Cleanups - string aggregates; json field names; excel sheet name cleansing; excel size limits; counts, etc

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -34,19 +34,23 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.DataSourceQueryInputCustomizer;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportViewCustomizer;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QFormulaException;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@@ -54,6 +58,9 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -79,6 +86,8 @@ import com.kingsrook.qqq.backend.core.utils.aggregates.InstantAggregates;
 import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates;
 import com.kingsrook.qqq.backend.core.utils.aggregates.LocalDateAggregates;
 import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates;
+import com.kingsrook.qqq.backend.core.utils.aggregates.StringAggregates;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -93,7 +102,7 @@ import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates;
  ** - summaries (like pivot tables, but called summary to avoid confusion with "native" pivot tables),
  ** - native pivot tables (not initially supported, due to lack of support in fastexcel...).
  *******************************************************************************/
-public class GenerateReportAction
+public class GenerateReportAction extends AbstractQActionFunction<ReportInput, ReportOutput>
 {
    private static final QLogger LOG = QLogger.getLogger(GenerateReportAction.class);
 
@@ -117,13 +126,16 @@ public class GenerateReportAction
    private List<QReportDataSource> dataSources;
    private List<QReportView>       views;
 
+   private Map<String, Integer> countByDataSource = new HashMap<>();
+
 
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void execute(ReportInput reportInput) throws QException
+   public ReportOutput execute(ReportInput reportInput) throws QException
    {
+      ReportOutput reportOutput = new ReportOutput();
       QReportMetaData report = getReportMetaData(reportInput);
 
       this.views = report.getViews();
@@ -203,21 +215,27 @@ public class GenerateReportAction
                ////////////////////////////////////////////////////////////////////////////////////
                // start the table-view (e.g., open this tab in xlsx) and then run the query-loop //
                ////////////////////////////////////////////////////////////////////////////////////
-               startTableView(reportInput, dataSource, dataSourceTableView);
+               startTableView(reportInput, dataSource, dataSourceTableView, reportFormat);
                gatherData(reportInput, dataSource, dataSourceTableView, dataSourceSummaryViews, dataSourceVariantViews);
             }
          }
       }
 
-      ////////////////////////////////////////
+      //////////////////////
       // add pivot sheets //
-      // todo - but, only for Excel, right? //
+      //////////////////////
-      ////////////////////////////////////////
       for(QReportView view : views)
       {
          if(view.getType().equals(ReportType.PIVOT))
          {
-            startTableView(reportInput, null, view);
+            if(reportFormat.getSupportsNativePivotTables())
+            {
+               startTableView(reportInput, null, view, reportFormat);
+            }
+            else
+            {
+               LOG.warn("Request to render a report with a PIVOT type view, for a format that does not support native pivot tables", logPair("reportFormat", reportFormat));
+            }
 
             //////////////////////////////////////////////////////////////////////////
             // there's no data to add to a pivot table, so nothing else to do here. //
@@ -227,6 +245,8 @@ public class GenerateReportAction
 
       outputSummaries(reportInput);
 
+      reportOutput.setTotalRecordCount(countByDataSource.values().stream().mapToInt(Integer::intValue).sum());
+
       reportStreamer.finish();
 
       try
@@ -237,6 +257,8 @@ public class GenerateReportAction
       {
          throw (new QReportingException("Error completing report", e));
       }
+
+      return (reportOutput);
    }
 
 
@@ -264,7 +286,7 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView) throws QException
+   private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView, ReportFormat reportFormat) throws QException
    {
       QMetaDataVariableInterpreter variableInterpreter = new QMetaDataVariableInterpreter();
       variableInterpreter.addValueMap("input", reportInput.getInputValues());
@@ -281,6 +303,8 @@ public class GenerateReportAction
          {
             joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), dataSource.getQueryFilter());
          }
+
+         countDataSourceRecords(reportInput, dataSource, reportFormat);
       }
 
       List<QFieldMetaData> fields = new ArrayList<>();
@@ -318,7 +342,36 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
+   private void countDataSourceRecords(ReportInput reportInput, QReportDataSource dataSource, ReportFormat reportFormat) throws QException
+   {
+      QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
+      setInputValuesInQueryFilter(reportInput, queryFilter);
+
+      CountInput countInput = new CountInput();
+      countInput.setTableName(dataSource.getSourceTable());
+      countInput.setFilter(queryFilter);
+      countInput.setQueryJoins(dataSource.getQueryJoins());
+      CountOutput countOutput = new CountAction().execute(countInput);
+
+      if(countOutput.getCount() != null)
+      {
+         countByDataSource.put(dataSource.getName(), countOutput.getCount());
+
+         if(reportFormat.getMaxRows() != null && countOutput.getCount() > reportFormat.getMaxRows())
+         {
+            throw (new QUserFacingException("The requested report would include more rows ("
+               + String.format("%,d", countOutput.getCount()) + ") than the maximum allowed ("
+               + String.format("%,d", reportFormat.getMaxRows()) + ") for the selected file format (" + reportFormat + ")."));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Integer gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
    {
       ////////////////////////////////////////////////////////////////////////////////////////
       // check if this view has a transform step - if so, set it up now and run its pre-run //
@@ -345,6 +398,9 @@ public class GenerateReportAction
       RunBackendStepInput   finalTransformStepInput  = transformStepInput;
       RunBackendStepOutput  finalTransformStepOutput = transformStepOutput;
 
+      String tableLabel = QContext.getQInstance().getTable(dataSource.getSourceTable()).getLabel();
+      AtomicInteger consumedCount = new AtomicInteger(0);
+
       /////////////////////////////////////////////////////////////////
       // run a record pipe loop, over the query for this data source //
       /////////////////////////////////////////////////////////////////
@@ -405,6 +461,17 @@ public class GenerateReportAction
             records = finalTransformStepOutput.getRecords();
          }
 
+         Integer total = countByDataSource.get(dataSource.getName());
+         if(total != null)
+         {
+            reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records", consumedCount.get() + 1, total);
+         }
+         else
+         {
+            reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records (" + String.format("%,d", consumedCount.get() + 1) + ")");
+         }
+         consumedCount.getAndAdd(records.size());
+
          return (consumeRecords(reportInput, dataSource, records, tableView, summaryViews, variantViews));
       });
 
@@ -415,6 +482,8 @@ public class GenerateReportAction
       {
          transformStep.postRun(new BackendStepPostRunInput(transformStepInput), new BackendStepPostRunOutput(transformStepOutput));
       }
+
+      return consumedCount.get();
    }
 
 
@@ -422,7 +491,7 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private Set<String> setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext)
+   private Set<String> setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext) throws QException
    {
       Set<String> fieldsToTranslatePossibleValues = new HashSet<>();
 
@@ -440,15 +509,16 @@ public class GenerateReportAction
             }
          }
 
-         for(String summaryField : CollectionUtils.nonNullList(view.getSummaryFields()))
+         for(String summaryFieldName : CollectionUtils.nonNullList(view.getSummaryFields()))
          {
             ///////////////////////////////////////////////////////////////////////////////
             // all pivotFields that are possible value sources are implicitly translated //
             ///////////////////////////////////////////////////////////////////////////////
-            QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable());
+            QTableMetaData    mainTable         = QContext.getQInstance().getTable(dataSource.getSourceTable());
-            if(table.getField(summaryField).getPossibleValueSourceName() != null)
+            FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(mainTable, summaryFieldName);
+            if(fieldAndJoinTable.field().getPossibleValueSourceName() != null)
             {
-               fieldsToTranslatePossibleValues.add(summaryField);
+               fieldsToTranslatePossibleValues.add(summaryFieldName);
             }
          }
       }
@@ -458,6 +528,32 @@ public class GenerateReportAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private FieldAndJoinTable getFieldAndJoinTable(QTableMetaData mainTable, String fieldName) throws QException
+   {
+      if(fieldName.indexOf('.') > -1)
+      {
+         String joinTableName = fieldName.replaceAll("\\..*", "");
+         String joinFieldName = fieldName.replaceAll(".*\\.", "");
+
+         QTableMetaData joinTable = QContext.getQInstance().getTable(joinTableName);
+         if(joinTable == null)
+         {
+            throw (new QException("Unrecognized join table name: " + joinTableName));
+         }
+
+         return new FieldAndJoinTable(joinTable.getField(joinFieldName), joinTable);
+      }
+      else
+      {
+         return new FieldAndJoinTable(mainTable.getField(fieldName), mainTable);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -550,24 +646,25 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List<QRecord> records, Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>>> aggregatesMap)
+   private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List<QRecord> records, Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>>> aggregatesMap) throws QException
    {
       Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>());
 
       for(QRecord record : records)
       {
          SummaryKey key = new SummaryKey();
-         for(String summaryField : view.getSummaryFields())
+         for(String summaryFieldName : view.getSummaryFields())
          {
-            Serializable summaryValue = record.getValue(summaryField);
+            FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName);
-            if(table.getField(summaryField).getPossibleValueSourceName() != null)
+            Serializable      summaryValue      = record.getValue(summaryFieldName);
+            if(fieldAndJoinTable.field().getPossibleValueSourceName() != null)
             {
                //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
                // so, this is kinda a thing - where we implicitly use possible-value labels (e.g., display values) for pivot fields... //
                //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               summaryValue = record.getDisplayValue(summaryField);
+               summaryValue = record.getDisplayValue(summaryFieldName);
             }
-            key.add(summaryField, summaryValue);
+            key.add(summaryFieldName, summaryValue);
 
             if(view.getIncludeSummarySubTotals() && key.getKeys().size() < view.getSummaryFields().size())
             {
@@ -588,7 +685,7 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates, SummaryKey key)
+   private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates, SummaryKey key) throws QException
    {
       Map<String, AggregatesInterface<?, ?>> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>());
       addRecordToAggregatesMap(table, record, keyAggregates);
@@ -599,44 +696,57 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map<String, AggregatesInterface<?, ?>> aggregatesMap)
+   private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map<String, AggregatesInterface<?, ?>> aggregatesMap) throws QException
    {
-      for(QFieldMetaData field : table.getFields().values())
+      //////////////////////////////////////////////////////////////////////////////////////
+      // todo - an optimization could be, to only compute aggregates that we'll need...   //
+      // Only if we measure and see this to be slow - it may be, lots of BigDecimal math? //
+      //////////////////////////////////////////////////////////////////////////////////////
+      for(String fieldName : record.getValues().keySet())
       {
+         FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, fieldName);
+         QFieldMetaData    field             = fieldAndJoinTable.field();
          if(StringUtils.hasContent(field.getPossibleValueSourceName()))
          {
-            continue;
+            @SuppressWarnings("unchecked")
+            AggregatesInterface<String, ?> fieldAggregates = (AggregatesInterface<String, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates());
+            fieldAggregates.add(record.getDisplayValue(fieldName));
          }
-
+         else if(field.getType().equals(QFieldType.INTEGER))
-         if(field.getType().equals(QFieldType.INTEGER))
          {
             @SuppressWarnings("unchecked")
-            AggregatesInterface<Integer, ?> fieldAggregates = (AggregatesInterface<Integer, ?>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new IntegerAggregates());
+            AggregatesInterface<Integer, ?> fieldAggregates = (AggregatesInterface<Integer, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new IntegerAggregates());
-            fieldAggregates.add(record.getValueInteger(field.getName()));
+            fieldAggregates.add(record.getValueInteger(fieldName));
          }
          else if(field.getType().equals(QFieldType.LONG))
          {
             @SuppressWarnings("unchecked")
-            AggregatesInterface<Long, ?> fieldAggregates = (AggregatesInterface<Long, ?>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new LongAggregates());
+            AggregatesInterface<Long, ?> fieldAggregates = (AggregatesInterface<Long, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LongAggregates());
-            fieldAggregates.add(record.getValueLong(field.getName()));
+            fieldAggregates.add(record.getValueLong(fieldName));
          }
          else if(field.getType().equals(QFieldType.DECIMAL))
          {
             @SuppressWarnings("unchecked")
-            AggregatesInterface<BigDecimal, ?> fieldAggregates = (AggregatesInterface<BigDecimal, ?>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new BigDecimalAggregates());
+            AggregatesInterface<BigDecimal, ?> fieldAggregates = (AggregatesInterface<BigDecimal, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new BigDecimalAggregates());
-            fieldAggregates.add(record.getValueBigDecimal(field.getName()));
+            fieldAggregates.add(record.getValueBigDecimal(fieldName));
          }
          else if(field.getType().equals(QFieldType.DATE_TIME))
          {
             @SuppressWarnings("unchecked")
-            AggregatesInterface<Instant, ?> fieldAggregates = (AggregatesInterface<Instant, ?>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new InstantAggregates());
+            AggregatesInterface<Instant, ?> fieldAggregates = (AggregatesInterface<Instant, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new InstantAggregates());
-            fieldAggregates.add(record.getValueInstant(field.getName()));
+            fieldAggregates.add(record.getValueInstant(fieldName));
          }
          else if(field.getType().equals(QFieldType.DATE))
          {
             @SuppressWarnings("unchecked")
-            AggregatesInterface<LocalDate, ?> fieldAggregates = (AggregatesInterface<LocalDate, ?>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new LocalDateAggregates());
+            AggregatesInterface<LocalDate, ?> fieldAggregates = (AggregatesInterface<LocalDate, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LocalDateAggregates());
-            fieldAggregates.add(record.getValueLocalDate(field.getName()));
+            fieldAggregates.add(record.getValueLocalDate(fieldName));
+         }
+         if(field.getType().isStringLike())
+         {
+            @SuppressWarnings("unchecked")
+            AggregatesInterface<String, ?> fieldAggregates = (AggregatesInterface<String, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates());
+            fieldAggregates.add(record.getValueString(fieldName));
          }
       }
    }
@@ -646,7 +756,7 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void outputSummaries(ReportInput reportInput) throws QReportingException, QFormulaException
+   private void outputSummaries(ReportInput reportInput) throws QException
    {
       List<QReportView> reportViews = views.stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList();
       for(QReportView view : reportViews)
@@ -719,13 +829,13 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private List<QFieldMetaData> getFields(QTableMetaData table, QReportView view)
+   private List<QFieldMetaData> getFields(QTableMetaData table, QReportView view) throws QException
    {
       List<QFieldMetaData> fields = new ArrayList<>();
-      for(String summaryField : view.getSummaryFields())
+      for(String summaryFieldName : view.getSummaryFields())
       {
-         QFieldMetaData field = table.getField(summaryField);
+         FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName);
-         fields.add(new QFieldMetaData(summaryField, field.getType()).withLabel(field.getLabel())); // todo do we need the type?  if so need table as input here
+         fields.add(new QFieldMetaData(summaryFieldName, fieldAndJoinTable.field().getType()).withLabel(fieldAndJoinTable.field().getLabel())); // todo do we need the type?  if so need table as input here
       }
       for(QReportField column : view.getColumns())
       {
@@ -982,4 +1092,10 @@ public class GenerateReportAction
    {
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable) {}
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java

@@ -30,6 +30,9 @@ import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
@@ -39,7 +42,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 
 
 /*******************************************************************************
@@ -64,6 +67,9 @@ public class JsonExportStreamer implements ExportStreamerInterface
    private byte[] indent       = new byte[0];
    private String indentString = "";
 
+   private Pattern                     colonLetterPattern    = Pattern.compile(":([A-Z]+)($|[A-Z][a-z])");
+   private Memoization<String, String> fieldLabelMemoization = new Memoization<>();
+
 
 
    /*******************************************************************************
@@ -232,8 +238,7 @@ public class JsonExportStreamer implements ExportStreamerInterface
          Map<String, Serializable> mapForJson = new LinkedHashMap<>();
          for(QFieldMetaData field : fields)
          {
-            String labelForJson = StringUtils.lcFirst(field.getLabel().replace(" ", ""));
+            mapForJson.put(getLabelForJson(field), qRecord.getValue(field.getName()));
-            mapForJson.put(labelForJson, qRecord.getValue(field.getName()));
          }
 
          String json = prettyPrint ? JsonUtils.toPrettyJson(mapForJson) : JsonUtils.toJson(mapForJson);
@@ -261,6 +266,73 @@ public class JsonExportStreamer implements ExportStreamerInterface
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   String getLabelForJson(QFieldMetaData field)
+   {
+      //////////////////////////////////////////////////////////////////////////
+      // memoize, to avoid running these regex/replacements millions of times //
+      //////////////////////////////////////////////////////////////////////////
+      Optional<String> result = fieldLabelMemoization.getResult(field.getName(), fieldName ->
+      {
+         String labelForJson = field.getLabel().replace(" ", "");
+
+         /////////////////////////////////////////////////////////////////////////////
+         // now fix up any field-name-parts after the table: portion of a join name //
+         // lineItem:SKU to become lineItem:sku                                     //
+         // parcel:SLAStatus to become parcel:slaStatus                             //
+         // order:Client to become order:client                                     //
+         /////////////////////////////////////////////////////////////////////////////
+         Matcher allCaps                          = Pattern.compile("^[A-Z]+$").matcher(labelForJson);
+         Matcher startsAllCapsThenNextWordMatcher = Pattern.compile("([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson);
+         Matcher startsOneCapMatcher              = Pattern.compile("([A-Z])(.*)").matcher(labelForJson);
+
+         if(allCaps.matches())
+         {
+            labelForJson = allCaps.replaceAll(m -> m.group().toLowerCase());
+         }
+         else if(startsAllCapsThenNextWordMatcher.matches())
+         {
+            labelForJson = startsAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2));
+         }
+         else if(startsOneCapMatcher.matches())
+         {
+            labelForJson = startsOneCapMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2));
+         }
+
+         /////////////////////////////////////////////////////////////////////////////
+         // now fix up any field-name-parts after the table: portion of a join name //
+         // lineItem:SKU to become lineItem:sku                                     //
+         // parcel:SLAStatus to become parcel:slaStatus                             //
+         // order:Client to become order:client                                     //
+         /////////////////////////////////////////////////////////////////////////////
+         Matcher colonThenAllCapsThenEndMatcher      = Pattern.compile("(.*:)([A-Z]+)$").matcher(labelForJson);
+         Matcher colonThenAllCapsThenNextWordMatcher = Pattern.compile("(.*:)([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson);
+         Matcher colonThenOneCapMatcher              = Pattern.compile("(.*:)([A-Z])(.*)").matcher(labelForJson);
+
+         if(colonThenAllCapsThenEndMatcher.matches())
+         {
+            labelForJson = colonThenAllCapsThenEndMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase());
+         }
+         else if(colonThenAllCapsThenNextWordMatcher.matches())
+         {
+            labelForJson = colonThenAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3));
+         }
+         else if(colonThenOneCapMatcher.matches())
+         {
+            labelForJson = colonThenOneCapMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3));
+         }
+
+         System.out.println("Label: " + labelForJson);
+         return (labelForJson);
+      });
+
+      return result.orElse(field.getName());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java

@@ -70,6 +70,7 @@ import org.apache.poi.ss.usermodel.DataConsolidateFunction;
 import org.apache.poi.ss.usermodel.DateUtil;
 import org.apache.poi.ss.util.AreaReference;
 import org.apache.poi.ss.util.CellReference;
+import org.apache.poi.ss.util.WorkbookUtil;
 import org.apache.poi.xssf.usermodel.XSSFCell;
 import org.apache.poi.xssf.usermodel.XSSFCellStyle;
 import org.apache.poi.xssf.usermodel.XSSFPivotTable;
@@ -151,6 +152,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
          for(QReportView view : views)
          {
             String label = Objects.requireNonNullElse(view.getLabel(), "Sheet " + sheetCounter);
+            label = WorkbookUtil.createSafeSheetName(label);
 
             /////////////////////////////////////////////////////////////////////////////////////////////
             // track the actually-used sheet labels (needed for referencing in pivot table generation) //
@@ -637,18 +639,6 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
       {
          throw (new QReportingException("Error adding totals row", e));
       }
-
-      /* todo
-      CellStyle totalsStyle = workbook.createCellStyle();
-      Font      font        = workbook.createFont();
-      font.setBold(true);
-      totalsStyle.setFont(font);
-      totalsStyle.setBorderTop(BorderStyle.THIN);
-      totalsStyle.setBorderTop(BorderStyle.THIN);
-      totalsStyle.setBorderBottom(BorderStyle.DOUBLE);
-
-      row.cellIterator().forEachRemaining(cell -> cell.setCellStyle(totalsStyle));
-       */
    }
 
 
@@ -666,9 +656,10 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
          //////////////////////////////////////////////
          closeLastSheetIfOpen();
 
-         /////////////////////////////
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // close the output stream //
+         // note - leave the zipOutputStream open.  It is a wrapper around the OutputStream we were given by the caller, //
-         /////////////////////////////
+         // so it is their responsibility to close that stream (which implicitly closes the zip, it appears)             //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
          zipOutputStream.close();
       }
       catch(Exception e)
@@ -784,7 +775,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
                  </cacheFields>
                </pivotCacheDefinition>
                """,
-            labelViewsByName.get(dataView.getName()),
+            StreamedPoiSheetWriter.cleanseValue(labelViewsByName.get(dataView.getName())),
             CellReference.convertNumToColString(dataView.getColumns().size() - 1),
             rowsPerView.get(dataView.getName()),
             dataView.getColumns().size(),

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedPoiSheetWriter.java

@@ -121,7 +121,7 @@ public class StreamedPoiSheetWriter
    /*******************************************************************************
     **
     *******************************************************************************/
-   private String cleanseValue(String value)
+   public static String cleanseValue(String value)
    {
       // todo - profile...
       if(xmlSpecialChars.matcher(value).find())

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java

@@ -32,7 +32,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
 
 
 /*******************************************************************************
- ** Input for an Export action
+ ** Input for a Report action
  *******************************************************************************/
 public class ReportInput extends AbstractTableActionInput
 {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportOutput.java

@@ -0,0 +1,67 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.reporting;
+
+
+import java.io.Serializable;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
+
+
+/*******************************************************************************
+ ** Output for a Report action
+ *******************************************************************************/
+public class ReportOutput extends AbstractActionOutput implements Serializable
+{
+   private Integer totalRecordCount;
+
+
+
+   /*******************************************************************************
+    ** Getter for totalRecordCount
+    *******************************************************************************/
+   public Integer getTotalRecordCount()
+   {
+      return (this.totalRecordCount);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for totalRecordCount
+    *******************************************************************************/
+   public void setTotalRecordCount(Integer totalRecordCount)
+   {
+      this.totalRecordCount = totalRecordCount;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for totalRecordCount
+    *******************************************************************************/
+   public ReportOutput withTotalRecordCount(Integer totalRecordCount)
+   {
+      this.totalRecordCount = totalRecordCount;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java

@@ -77,6 +77,7 @@ public class SavedReportToReportMetaDataAdapter
          QInstance qInstance = QContext.getQInstance();
 
          QReportMetaData reportMetaData = new QReportMetaData();
+         reportMetaData.setName("savedReport:" + savedReport.getId());
          reportMetaData.setLabel(savedReport.getLabel());
 
          /////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/StringAggregates.java

@@ -0,0 +1,121 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.utils.aggregates;
+
+
+/*******************************************************************************
+ ** String version of data aggregator
+ *******************************************************************************/
+public class StringAggregates implements AggregatesInterface<String, String>
+{
+   private int count = 0;
+
+   private String min;
+   private String max;
+
+
+
+   /*******************************************************************************
+    ** Add a new value to this aggregate set
+    *******************************************************************************/
+   public void add(String input)
+   {
+      if(input == null)
+      {
+         return;
+      }
+
+      count++;
+
+      if(min == null || input.compareTo(min) < 0)
+      {
+         min = input;
+      }
+
+      if(max == null || input.compareTo(max) > 0)
+      {
+         max = input;
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public int getCount()
+   {
+      return (count);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getSum()
+   {
+      //////////////////////////////////////
+      // sum of string doesn't make sense //
+      //////////////////////////////////////
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getMin()
+   {
+      return (min);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getMax()
+   {
+      return (max);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getAverage()
+   {
+      ///////////////////////////////////////
+      // average string doesn't make sense //
+      ///////////////////////////////////////
+      return (null);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamerTest.java

@@ -0,0 +1,54 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.util.function.Function;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for JsonExportStreamer 
+ *******************************************************************************/
+class JsonExportStreamerTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test()
+   {
+      Function<String, String> runOne = label -> new JsonExportStreamer().getLabelForJson(new QFieldMetaData("test", QFieldType.STRING).withLabel(label));
+      assertEquals("sku", runOne.apply("SKU"));
+      assertEquals("clientName", runOne.apply("Client Name"));
+      assertEquals("slaStatus", runOne.apply("SLA Status"));
+      assertEquals("lineItem:sku", runOne.apply("Line Item: SKU"));
+      assertEquals("parcel:slaStatus", runOne.apply("Parcel: SLA Status"));
+      assertEquals("order:client", runOne.apply("Order: Client"));
+   }
+
+}

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStream.java

@@ -36,6 +36,8 @@ import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.PutObjectResult;
 import com.amazonaws.services.s3.model.UploadPartRequest;
 import com.amazonaws.services.s3.model.UploadPartResult;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -45,6 +47,8 @@ import com.amazonaws.services.s3.model.UploadPartResult;
  *******************************************************************************/
 public class S3UploadOutputStream extends OutputStream
 {
+   private static final QLogger LOG = QLogger.getLogger(S3UploadOutputStream.class);
+
    private final AmazonS3 amazonS3;
    private final String   bucketName;
    private final String   key;
@@ -55,6 +59,8 @@ public class S3UploadOutputStream extends OutputStream
    private InitiateMultipartUploadResult initiateMultipartUploadResult = null;
    private List<UploadPartResult>        uploadPartResultList          = null;
 
+   private boolean isClosed = false;
+
 
 
    /*******************************************************************************
@@ -96,10 +102,12 @@ public class S3UploadOutputStream extends OutputStream
          //////////////////////////////////////////
          if(initiateMultipartUploadResult == null)
          {
+            LOG.info("Initiating a multipart upload", logPair("key", key));
             initiateMultipartUploadResult = amazonS3.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucketName, key));
             uploadPartResultList = new ArrayList<>();
          }
 
+         LOG.info("Uploading a part", logPair("key", key), logPair("partNumber", uploadPartResultList.size() + 1));
          UploadPartRequest uploadPartRequest = new UploadPartRequest()
             .withUploadId(initiateMultipartUploadResult.getUploadId())
             .withPartNumber(uploadPartResultList.size() + 1)
@@ -130,7 +138,6 @@ public class S3UploadOutputStream extends OutputStream
       while(bytesToWrite > buffer.length - offset)
       {
          int size = buffer.length - offset;
-         // System.out.println("A:copy " + size + " bytes from source[" + off + "] to dest[" + offset + "]");
          System.arraycopy(b, off, buffer, offset, size);
          offset = buffer.length;
          uploadIfNeeded();
@@ -139,7 +146,6 @@ public class S3UploadOutputStream extends OutputStream
       }
 
       int size = len - off;
-      // System.out.println("B:copy " + size + " bytes from source[" + off + "] to dest[" + offset + "]");
       System.arraycopy(b, off, buffer, offset, size);
       offset += size;
       uploadIfNeeded();
@@ -153,6 +159,12 @@ public class S3UploadOutputStream extends OutputStream
    @Override
    public void close() throws IOException
    {
+      if(isClosed)
+      {
+         LOG.debug("Redundant call to close an already-closed S3UploadOutputStream.  Returning with noop.", logPair("key", key));
+         return;
+      }
+
       if(initiateMultipartUploadResult != null)
       {
          if(offset > 0)
@@ -160,6 +172,7 @@ public class S3UploadOutputStream extends OutputStream
             //////////////////////////////////////////////////
             // if there's a final part to upload, do it now //
             //////////////////////////////////////////////////
+            LOG.info("Uploading a part", logPair("key", key), logPair("isFinalPart", true), logPair("partNumber", uploadPartResultList.size() + 1));
             UploadPartRequest uploadPartRequest = new UploadPartRequest()
                .withUploadId(initiateMultipartUploadResult.getUploadId())
                .withPartNumber(uploadPartResultList.size() + 1)
@@ -179,10 +192,13 @@ public class S3UploadOutputStream extends OutputStream
       }
       else
       {
+         LOG.info("Putting object (non-multipart)", logPair("key", key), logPair("length", offset));
          ObjectMetadata objectMetadata = new ObjectMetadata();
          objectMetadata.setContentLength(offset);
          PutObjectResult putObjectResult = amazonS3.putObject(bucketName, key, new ByteArrayInputStream(buffer, 0, offset), objectMetadata);
       }
+
+      isClosed = true;
    }
 
 }

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.module.rdbms.reporting;
 
 
 import java.io.ByteArrayOutputStream;
+import java.io.FileOutputStream;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
@@ -38,6 +39,10 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableFunction;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@@ -59,6 +64,8 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryStorageAction;
 import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.RenderSavedReportMetaDataProducer;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.LocalMacDevUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.module.rdbms.TestUtils;
 import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest;
 import org.apache.commons.io.IOUtils;
@@ -215,23 +222,39 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest
    }
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
-   private List<String> runSavedReportForCSV(SavedReport newSavedReport) throws Exception
+   private RunProcessOutput runSavedReport(SavedReport savedReport, ReportFormatPossibleValueEnum reportFormat) throws Exception
    {
-      newSavedReport.setLabel("Test Report");
+      savedReport.setLabel("Test Report");
       QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
-      new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null);
 
-      QRecord savedReport = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(newSavedReport)).getRecords().get(0);
+      if(QContext.getQInstance().getTable(SavedReport.TABLE_NAME) == null)
+      {
+         new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null);
+      }
+
+      QRecord savedReportRecord = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(savedReport)).getRecords().get(0);
 
       RunProcessInput input = new RunProcessInput();
       input.setProcessName(RenderSavedReportMetaDataProducer.NAME);
       input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
-      input.setCallback(QProcessCallbackFactory.forRecord(savedReport));
+      input.setCallback(QProcessCallbackFactory.forRecord(savedReportRecord));
-      input.addValue("reportFormat", ReportFormatPossibleValueEnum.CSV.getPossibleValueId());
+      input.addValue("reportFormat", reportFormat);
       RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
+      return (runProcessOutput);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<String> runSavedReportForCSV(SavedReport savedReport) throws Exception
+   {
+      RunProcessOutput runProcessOutput = runSavedReport(savedReport, ReportFormatPossibleValueEnum.CSV);
 
       String      storageTableName = runProcessOutput.getValueString("storageTableName");
       String      storageReference = runProcessOutput.getValueString("storageReference");
@@ -293,6 +316,7 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest
    }
 
 
+
    /*******************************************************************************
     ** in here, by potentially ambiguous, we mean where there are possible joins
     ** between the order and orderInstructions tables.
@@ -341,7 +365,6 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest
          """.trim(), lines.get(1));
    }
 
-   // todo - similar to above, but w/o selecting, only filtering
 
 
    /*******************************************************************************
@@ -369,6 +392,51 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSavedReportWithPivotsFromJoinTable() throws Exception
+   {
+      SavedReport savedReport = new SavedReport()
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withColumnsJson(JsonUtils.toJson(new ReportColumns()
+            .withColumn("id")
+            .withColumn("item.storeId")
+            .withColumn("item.description")))
+         .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()))
+         .withPivotTableJson(JsonUtils.toJson(new PivotTableDefinition()
+            .withRow(new PivotTableGroupBy().withFieldName("item.storeId"))
+            .withValue(new PivotTableValue().withFieldName("item.description").withFunction(PivotTableFunction.COUNT))));
+
+      //////////////////////////////////////////////
+      // make sure we can render xlsx w/o a crash //
+      //////////////////////////////////////////////
+      RunProcessOutput runProcessOutput = runSavedReport(savedReport, ReportFormatPossibleValueEnum.XLSX);
+      String           storageTableName = runProcessOutput.getValueString("storageTableName");
+      String           storageReference = runProcessOutput.getValueString("storageReference");
+      InputStream      inputStream      = new MemoryStorageAction().getInputStream(new StorageInput(storageTableName).withReference(storageReference));
+
+      String path = "/tmp/pivot.xlsx";
+      inputStream.transferTo(new FileOutputStream(path));
+      // LocalMacDevUtils.mayOpenFiles = true;
+      LocalMacDevUtils.openFile(path);
+
+      ///////////////////////////////////////////////////////
+      // render as csv too - and assert about those values //
+      ///////////////////////////////////////////////////////
+      List<String> csv = runSavedReportForCSV(savedReport);
+      System.out.println(StringUtils.join("\n", csv));
+      assertEquals("""
+         "Store","Count Of Item: Description\"""", csv.get(0));
+      assertEquals("""
+         "Q-Mart","4\"""", csv.get(1));
+      assertEquals("""
+         "Totals","11\"""", csv.get(4));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/