QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-881 - Cleanups - string aggregates; json field names; excel sheet name cleansing; excel size limits; counts, etc

Browse Source
This commit is contained in:
darin.kelkhoff
2024-04-02 15:43:59 -05:00
parent 7c8ef52af9
commit 4dadff7fc2
12 changed files with 584 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java
View File

@ -34,19 +34,23 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.reporting.customizers.DataSourceQueryInputCustomizer;
import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportViewCustomizer;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QFormulaException;
import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@ -54,6 +58,9 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp
import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@ -79,6 +86,8 @@ import com.kingsrook.qqq.backend.core.utils.aggregates.InstantAggregates;
import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates;
import com.kingsrook.qqq.backend.core.utils.aggregates.LocalDateAggregates;
import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates;
import com.kingsrook.qqq.backend.core.utils.aggregates.StringAggregates;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@ -93,7 +102,7 @@ import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates;
** - summaries (like pivot tables, but called summary to avoid confusion with "native" pivot tables),
** - native pivot tables (not initially supported, due to lack of support in fastexcel...).
*******************************************************************************/
public class GenerateReportAction
public class GenerateReportAction extends AbstractQActionFunction<ReportInput, ReportOutput>
{
private static final QLogger LOG = QLogger.getLogger(GenerateReportAction.class);
@ -117,13 +126,16 @@ public class GenerateReportAction
private List<QReportDataSource> dataSources;
private List<QReportView> views;
private Map<String, Integer> countByDataSource = new HashMap<>();
/*******************************************************************************
**
*******************************************************************************/
public void execute(ReportInput reportInput) throws QException
public ReportOutput execute(ReportInput reportInput) throws QException
{
ReportOutput reportOutput = new ReportOutput();
QReportMetaData report = getReportMetaData(reportInput);
this.views = report.getViews();
@ -203,21 +215,27 @@ public class GenerateReportAction
////////////////////////////////////////////////////////////////////////////////////
// start the table-view (e.g., open this tab in xlsx) and then run the query-loop //
////////////////////////////////////////////////////////////////////////////////////
startTableView(reportInput, dataSource, dataSourceTableView);
startTableView(reportInput, dataSource, dataSourceTableView, reportFormat);
gatherData(reportInput, dataSource, dataSourceTableView, dataSourceSummaryViews, dataSourceVariantViews);
}
}
}
////////////////////////////////////////
// add pivot sheets //
// todo - but, only for Excel, right? //
////////////////////////////////////////
//////////////////////
// add pivot sheets //
//////////////////////
for(QReportView view : views)
{
if(view.getType().equals(ReportType.PIVOT))
{
startTableView(reportInput, null, view);
if(reportFormat.getSupportsNativePivotTables())
{
startTableView(reportInput, null, view, reportFormat);
}
else
{
LOG.warn("Request to render a report with a PIVOT type view, for a format that does not support native pivot tables", logPair("reportFormat", reportFormat));
}
//////////////////////////////////////////////////////////////////////////
// there's no data to add to a pivot table, so nothing else to do here. //
@ -227,6 +245,8 @@ public class GenerateReportAction
outputSummaries(reportInput);
reportOutput.setTotalRecordCount(countByDataSource.values().stream().mapToInt(Integer::intValue).sum());
reportStreamer.finish();
try
@ -237,6 +257,8 @@ public class GenerateReportAction
{
throw (new QReportingException("Error completing report", e));
}
return (reportOutput);
}
@ -264,7 +286,7 @@ public class GenerateReportAction
/*******************************************************************************
**
*******************************************************************************/
private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView) throws QException
private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView, ReportFormat reportFormat) throws QException
{
QMetaDataVariableInterpreter variableInterpreter = new QMetaDataVariableInterpreter();
variableInterpreter.addValueMap("input", reportInput.getInputValues());
@ -281,6 +303,8 @@ public class GenerateReportAction
{
joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), dataSource.getQueryFilter());
}
countDataSourceRecords(reportInput, dataSource, reportFormat);
}
List<QFieldMetaData> fields = new ArrayList<>();
@ -318,7 +342,36 @@ public class GenerateReportAction
/*******************************************************************************
**
*******************************************************************************/
private void gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
private void countDataSourceRecords(ReportInput reportInput, QReportDataSource dataSource, ReportFormat reportFormat) throws QException
{
QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
setInputValuesInQueryFilter(reportInput, queryFilter);
CountInput countInput = new CountInput();
countInput.setTableName(dataSource.getSourceTable());
countInput.setFilter(queryFilter);
countInput.setQueryJoins(dataSource.getQueryJoins());
CountOutput countOutput = new CountAction().execute(countInput);
if(countOutput.getCount() != null)
{
countByDataSource.put(dataSource.getName(), countOutput.getCount());
if(reportFormat.getMaxRows() != null && countOutput.getCount() > reportFormat.getMaxRows())
{
throw (new QUserFacingException("The requested report would include more rows ("
+ String.format("%,d", countOutput.getCount()) + ") than the maximum allowed ("
+ String.format("%,d", reportFormat.getMaxRows()) + ") for the selected file format (" + reportFormat + ")."));
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private Integer gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
{
////////////////////////////////////////////////////////////////////////////////////////
// check if this view has a transform step - if so, set it up now and run its pre-run //
@ -345,6 +398,9 @@ public class GenerateReportAction
RunBackendStepInput finalTransformStepInput = transformStepInput;
RunBackendStepOutput finalTransformStepOutput = transformStepOutput;
String tableLabel = QContext.getQInstance().getTable(dataSource.getSourceTable()).getLabel();
AtomicInteger consumedCount = new AtomicInteger(0);
/////////////////////////////////////////////////////////////////
// run a record pipe loop, over the query for this data source //
/////////////////////////////////////////////////////////////////
@ -405,6 +461,17 @@ public class GenerateReportAction
records = finalTransformStepOutput.getRecords();
}
Integer total = countByDataSource.get(dataSource.getName());
if(total != null)
{
reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records", consumedCount.get() + 1, total);
}
else
{
reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records (" + String.format("%,d", consumedCount.get() + 1) + ")");
}
consumedCount.getAndAdd(records.size());
return (consumeRecords(reportInput, dataSource, records, tableView, summaryViews, variantViews));
});
@ -415,6 +482,8 @@ public class GenerateReportAction
{
transformStep.postRun(new BackendStepPostRunInput(transformStepInput), new BackendStepPostRunOutput(transformStepOutput));
}
return consumedCount.get();
}
@ -422,7 +491,7 @@ public class GenerateReportAction
/*******************************************************************************
**
*******************************************************************************/
private Set<String> setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext)
private Set<String> setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext) throws QException
{
Set<String> fieldsToTranslatePossibleValues = new HashSet<>();
@ -440,15 +509,16 @@ public class GenerateReportAction
}
}
for(String summaryField : CollectionUtils.nonNullList(view.getSummaryFields()))
for(String summaryFieldName : CollectionUtils.nonNullList(view.getSummaryFields()))
{
///////////////////////////////////////////////////////////////////////////////
// all pivotFields that are possible value sources are implicitly translated //
///////////////////////////////////////////////////////////////////////////////
QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable());
if(table.getField(summaryField).getPossibleValueSourceName() != null)
QTableMetaData mainTable = QContext.getQInstance().getTable(dataSource.getSourceTable());
FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(mainTable, summaryFieldName);
if(fieldAndJoinTable.field().getPossibleValueSourceName() != null)
{
fieldsToTranslatePossibleValues.add(summaryField);
fieldsToTranslatePossibleValues.add(summaryFieldName);
}
}
}
@ -458,6 +528,32 @@ public class GenerateReportAction
/*******************************************************************************
**
*******************************************************************************/
private FieldAndJoinTable getFieldAndJoinTable(QTableMetaData mainTable, String fieldName) throws QException
{
if(fieldName.indexOf('.') > -1)
{
String joinTableName = fieldName.replaceAll("\\..*", "");
String joinFieldName = fieldName.replaceAll(".*\\.", "");
QTableMetaData joinTable = QContext.getQInstance().getTable(joinTableName);
if(joinTable == null)
{
throw (new QException("Unrecognized join table name: " + joinTableName));
}
return new FieldAndJoinTable(joinTable.getField(joinFieldName), joinTable);
}
else
{
return new FieldAndJoinTable(mainTable.getField(fieldName), mainTable);
}
}
/*******************************************************************************
**
*******************************************************************************/
@ -550,24 +646,25 @@ public class GenerateReportAction
/*******************************************************************************
**
*******************************************************************************/
private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List<QRecord> records, Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>>> aggregatesMap)
private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List<QRecord> records, Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>>> aggregatesMap) throws QException
{
Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>());
for(QRecord record : records)
{
SummaryKey key = new SummaryKey();
for(String summaryField : view.getSummaryFields())
for(String summaryFieldName : view.getSummaryFields())
{
Serializable summaryValue = record.getValue(summaryField);
if(table.getField(summaryField).getPossibleValueSourceName() != null)
FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName);
Serializable summaryValue = record.getValue(summaryFieldName);
if(fieldAndJoinTable.field().getPossibleValueSourceName() != null)
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// so, this is kinda a thing - where we implicitly use possible-value labels (e.g., display values) for pivot fields... //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
summaryValue = record.getDisplayValue(summaryField);
summaryValue = record.getDisplayValue(summaryFieldName);
}
key.add(summaryField, summaryValue);
key.add(summaryFieldName, summaryValue);
if(view.getIncludeSummarySubTotals() && key.getKeys().size() < view.getSummaryFields().size())
{
@ -588,7 +685,7 @@ public class GenerateReportAction
/*******************************************************************************
**
*******************************************************************************/
private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates, SummaryKey key)
private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates, SummaryKey key) throws QException
{
Map<String, AggregatesInterface<?, ?>> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>());
addRecordToAggregatesMap(table, record, keyAggregates);
@ -599,44 +696,57 @@ public class GenerateReportAction
/*******************************************************************************
**
*******************************************************************************/
private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map<String, AggregatesInterface<?, ?>> aggregatesMap)
private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map<String, AggregatesInterface<?, ?>> aggregatesMap) throws QException
{
for(QFieldMetaData field : table.getFields().values())
//////////////////////////////////////////////////////////////////////////////////////
// todo - an optimization could be, to only compute aggregates that we'll need... //
// Only if we measure and see this to be slow - it may be, lots of BigDecimal math? //
//////////////////////////////////////////////////////////////////////////////////////
for(String fieldName : record.getValues().keySet())
{
FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, fieldName);
QFieldMetaData field = fieldAndJoinTable.field();
if(StringUtils.hasContent(field.getPossibleValueSourceName()))
{
continue;
@SuppressWarnings("unchecked")
AggregatesInterface<String, ?> fieldAggregates = (AggregatesInterface<String, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates());
fieldAggregates.add(record.getDisplayValue(fieldName));
}
if(field.getType().equals(QFieldType.INTEGER))
else if(field.getType().equals(QFieldType.INTEGER))
{
@SuppressWarnings("unchecked")
AggregatesInterface<Integer, ?> fieldAggregates = (AggregatesInterface<Integer, ?>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new IntegerAggregates());
fieldAggregates.add(record.getValueInteger(field.getName()));
AggregatesInterface<Integer, ?> fieldAggregates = (AggregatesInterface<Integer, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new IntegerAggregates());
fieldAggregates.add(record.getValueInteger(fieldName));
}
else if(field.getType().equals(QFieldType.LONG))
{
@SuppressWarnings("unchecked")
AggregatesInterface<Long, ?> fieldAggregates = (AggregatesInterface<Long, ?>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new LongAggregates());
fieldAggregates.add(record.getValueLong(field.getName()));
AggregatesInterface<Long, ?> fieldAggregates = (AggregatesInterface<Long, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LongAggregates());
fieldAggregates.add(record.getValueLong(fieldName));
}
else if(field.getType().equals(QFieldType.DECIMAL))
{
@SuppressWarnings("unchecked")
AggregatesInterface<BigDecimal, ?> fieldAggregates = (AggregatesInterface<BigDecimal, ?>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new BigDecimalAggregates());
fieldAggregates.add(record.getValueBigDecimal(field.getName()));
AggregatesInterface<BigDecimal, ?> fieldAggregates = (AggregatesInterface<BigDecimal, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new BigDecimalAggregates());
fieldAggregates.add(record.getValueBigDecimal(fieldName));
}
else if(field.getType().equals(QFieldType.DATE_TIME))
{
@SuppressWarnings("unchecked")
AggregatesInterface<Instant, ?> fieldAggregates = (AggregatesInterface<Instant, ?>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new InstantAggregates());
fieldAggregates.add(record.getValueInstant(field.getName()));
AggregatesInterface<Instant, ?> fieldAggregates = (AggregatesInterface<Instant, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new InstantAggregates());
fieldAggregates.add(record.getValueInstant(fieldName));
}
else if(field.getType().equals(QFieldType.DATE))
{
@SuppressWarnings("unchecked")
AggregatesInterface<LocalDate, ?> fieldAggregates = (AggregatesInterface<LocalDate, ?>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new LocalDateAggregates());
fieldAggregates.add(record.getValueLocalDate(field.getName()));
AggregatesInterface<LocalDate, ?> fieldAggregates = (AggregatesInterface<LocalDate, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LocalDateAggregates());
fieldAggregates.add(record.getValueLocalDate(fieldName));
}
if(field.getType().isStringLike())
{
@SuppressWarnings("unchecked")
AggregatesInterface<String, ?> fieldAggregates = (AggregatesInterface<String, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates());
fieldAggregates.add(record.getValueString(fieldName));
}
}
}
@ -646,7 +756,7 @@ public class GenerateReportAction
/*******************************************************************************
**
*******************************************************************************/
private void outputSummaries(ReportInput reportInput) throws QReportingException, QFormulaException
private void outputSummaries(ReportInput reportInput) throws QException
{
List<QReportView> reportViews = views.stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList();
for(QReportView view : reportViews)
@ -719,13 +829,13 @@ public class GenerateReportAction
/*******************************************************************************
**
*******************************************************************************/
private List<QFieldMetaData> getFields(QTableMetaData table, QReportView view)
private List<QFieldMetaData> getFields(QTableMetaData table, QReportView view) throws QException
{
List<QFieldMetaData> fields = new ArrayList<>();
for(String summaryField : view.getSummaryFields())
for(String summaryFieldName : view.getSummaryFields())
{
QFieldMetaData field = table.getField(summaryField);
fields.add(new QFieldMetaData(summaryField, field.getType()).withLabel(field.getLabel())); // todo do we need the type? if so need table as input here
FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName);
fields.add(new QFieldMetaData(summaryFieldName, fieldAndJoinTable.field().getType()).withLabel(fieldAndJoinTable.field().getLabel())); // todo do we need the type? if so need table as input here
}
for(QReportField column : view.getColumns())
{
@ -982,4 +1092,10 @@ public class GenerateReportAction
{
}
/*******************************************************************************
**
*******************************************************************************/
private record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable) {}
}

78
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java
View File

@ -30,6 +30,9 @@ import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
@ -39,7 +42,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
@ -64,6 +67,9 @@ public class JsonExportStreamer implements ExportStreamerInterface
private byte[] indent = new byte[0];
private String indentString = "";
private Pattern colonLetterPattern = Pattern.compile(":([A-Z]+)($|[A-Z][a-z])");
private Memoization<String, String> fieldLabelMemoization = new Memoization<>();
/*******************************************************************************
@ -232,8 +238,7 @@ public class JsonExportStreamer implements ExportStreamerInterface
Map<String, Serializable> mapForJson = new LinkedHashMap<>();
for(QFieldMetaData field : fields)
{
String labelForJson = StringUtils.lcFirst(field.getLabel().replace(" ", ""));
mapForJson.put(labelForJson, qRecord.getValue(field.getName()));
mapForJson.put(getLabelForJson(field), qRecord.getValue(field.getName()));
}
String json = prettyPrint ? JsonUtils.toPrettyJson(mapForJson) : JsonUtils.toJson(mapForJson);
@ -261,6 +266,73 @@ public class JsonExportStreamer implements ExportStreamerInterface
/*******************************************************************************
**
*******************************************************************************/
String getLabelForJson(QFieldMetaData field)
{
//////////////////////////////////////////////////////////////////////////
// memoize, to avoid running these regex/replacements millions of times //
//////////////////////////////////////////////////////////////////////////
Optional<String> result = fieldLabelMemoization.getResult(field.getName(), fieldName ->
{
String labelForJson = field.getLabel().replace(" ", "");
/////////////////////////////////////////////////////////////////////////////
// now fix up any field-name-parts after the table: portion of a join name //
// lineItem:SKU to become lineItem:sku //
// parcel:SLAStatus to become parcel:slaStatus //
// order:Client to become order:client //
/////////////////////////////////////////////////////////////////////////////
Matcher allCaps = Pattern.compile("^[A-Z]+$").matcher(labelForJson);
Matcher startsAllCapsThenNextWordMatcher = Pattern.compile("([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson);
Matcher startsOneCapMatcher = Pattern.compile("([A-Z])(.*)").matcher(labelForJson);
if(allCaps.matches())
{
labelForJson = allCaps.replaceAll(m -> m.group().toLowerCase());
}
else if(startsAllCapsThenNextWordMatcher.matches())
{
labelForJson = startsAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2));
}
else if(startsOneCapMatcher.matches())
{
labelForJson = startsOneCapMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2));
}
/////////////////////////////////////////////////////////////////////////////
// now fix up any field-name-parts after the table: portion of a join name //
// lineItem:SKU to become lineItem:sku //
// parcel:SLAStatus to become parcel:slaStatus //
// order:Client to become order:client //
/////////////////////////////////////////////////////////////////////////////
Matcher colonThenAllCapsThenEndMatcher = Pattern.compile("(.*:)([A-Z]+)$").matcher(labelForJson);
Matcher colonThenAllCapsThenNextWordMatcher = Pattern.compile("(.*:)([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson);
Matcher colonThenOneCapMatcher = Pattern.compile("(.*:)([A-Z])(.*)").matcher(labelForJson);
if(colonThenAllCapsThenEndMatcher.matches())
{
labelForJson = colonThenAllCapsThenEndMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase());
}
else if(colonThenAllCapsThenNextWordMatcher.matches())
{
labelForJson = colonThenAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3));
}
else if(colonThenOneCapMatcher.matches())
{
labelForJson = colonThenOneCapMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3));
}
System.out.println("Label: " + labelForJson);
return (labelForJson);
});
return result.orElse(field.getName());
}
/*******************************************************************************
**
*******************************************************************************/

23
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java
View File

@ -70,6 +70,7 @@ import org.apache.poi.ss.usermodel.DataConsolidateFunction;
import org.apache.poi.ss.usermodel.DateUtil;
import org.apache.poi.ss.util.AreaReference;
import org.apache.poi.ss.util.CellReference;
import org.apache.poi.ss.util.WorkbookUtil;
import org.apache.poi.xssf.usermodel.XSSFCell;
import org.apache.poi.xssf.usermodel.XSSFCellStyle;
import org.apache.poi.xssf.usermodel.XSSFPivotTable;
@ -151,6 +152,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
for(QReportView view : views)
{
String label = Objects.requireNonNullElse(view.getLabel(), "Sheet " + sheetCounter);
label = WorkbookUtil.createSafeSheetName(label);
/////////////////////////////////////////////////////////////////////////////////////////////
// track the actually-used sheet labels (needed for referencing in pivot table generation) //
@ -637,18 +639,6 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
{
throw (new QReportingException("Error adding totals row", e));
}
/* todo
CellStyle totalsStyle = workbook.createCellStyle();
Font font = workbook.createFont();
font.setBold(true);
totalsStyle.setFont(font);
totalsStyle.setBorderTop(BorderStyle.THIN);
totalsStyle.setBorderTop(BorderStyle.THIN);
totalsStyle.setBorderBottom(BorderStyle.DOUBLE);
row.cellIterator().forEachRemaining(cell -> cell.setCellStyle(totalsStyle));
*/
}
@ -666,9 +656,10 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
//////////////////////////////////////////////
closeLastSheetIfOpen();
/////////////////////////////
// close the output stream //
/////////////////////////////
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// note - leave the zipOutputStream open. It is a wrapper around the OutputStream we were given by the caller, //
// so it is their responsibility to close that stream (which implicitly closes the zip, it appears) //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////
zipOutputStream.close();
}
catch(Exception e)
@ -784,7 +775,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
</cacheFields>
</pivotCacheDefinition>
""",
labelViewsByName.get(dataView.getName()),
StreamedPoiSheetWriter.cleanseValue(labelViewsByName.get(dataView.getName())),
CellReference.convertNumToColString(dataView.getColumns().size() - 1),
rowsPerView.get(dataView.getName()),
dataView.getColumns().size(),

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedPoiSheetWriter.java
View File

@ -121,7 +121,7 @@ public class StreamedPoiSheetWriter
/*******************************************************************************
**
*******************************************************************************/
private String cleanseValue(String value)
public static String cleanseValue(String value)
{
// todo - profile...
if(xmlSpecialChars.matcher(value).find())

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java
View File

@ -32,7 +32,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
/*******************************************************************************
** Input for an Export action
** Input for a Report action
*******************************************************************************/
public class ReportInput extends AbstractTableActionInput
{

67
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportOutput.java Normal file
View File

@ -0,0 +1,67 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.actions.reporting;
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
/*******************************************************************************
** Output for a Report action
*******************************************************************************/
public class ReportOutput extends AbstractActionOutput implements Serializable
{
private Integer totalRecordCount;
/*******************************************************************************
** Getter for totalRecordCount
*******************************************************************************/
public Integer getTotalRecordCount()
{
return (this.totalRecordCount);
}
/*******************************************************************************
** Setter for totalRecordCount
*******************************************************************************/
public void setTotalRecordCount(Integer totalRecordCount)
{
this.totalRecordCount = totalRecordCount;
}
/*******************************************************************************
** Fluent setter for totalRecordCount
*******************************************************************************/
public ReportOutput withTotalRecordCount(Integer totalRecordCount)
{
this.totalRecordCount = totalRecordCount;
return (this);
}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java
View File

@ -77,6 +77,7 @@ public class SavedReportToReportMetaDataAdapter
QInstance qInstance = QContext.getQInstance();
QReportMetaData reportMetaData = new QReportMetaData();
reportMetaData.setName("savedReport:" + savedReport.getId());
reportMetaData.setLabel(savedReport.getLabel());
/////////////////////////////////////////////////////

121
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/StringAggregates.java Normal file
View File

@ -0,0 +1,121 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.utils.aggregates;
/*******************************************************************************
** String version of data aggregator
*******************************************************************************/
public class StringAggregates implements AggregatesInterface<String, String>
{
private int count = 0;
private String min;
private String max;
/*******************************************************************************
** Add a new value to this aggregate set
*******************************************************************************/
public void add(String input)
{
if(input == null)
{
return;
}
count++;
if(min == null || input.compareTo(min) < 0)
{
min = input;
}
if(max == null || input.compareTo(max) > 0)
{
max = input;
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public int getCount()
{
return (count);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String getSum()
{
//////////////////////////////////////
// sum of string doesn't make sense //
//////////////////////////////////////
return (null);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String getMin()
{
return (min);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String getMax()
{
return (max);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String getAverage()
{
///////////////////////////////////////
// average string doesn't make sense //
///////////////////////////////////////
return (null);
}
}

54
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamerTest.java Normal file
View File

@ -0,0 +1,54 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.reporting;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for JsonExportStreamer
*******************************************************************************/
class JsonExportStreamerTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test()
{
Function<String, String> runOne = label -> new JsonExportStreamer().getLabelForJson(new QFieldMetaData("test", QFieldType.STRING).withLabel(label));
assertEquals("sku", runOne.apply("SKU"));
assertEquals("clientName", runOne.apply("Client Name"));
assertEquals("slaStatus", runOne.apply("SLA Status"));
assertEquals("lineItem:sku", runOne.apply("Line Item: SKU"));
assertEquals("parcel:slaStatus", runOne.apply("Parcel: SLA Status"));
assertEquals("order:client", runOne.apply("Order: Client"));
}
}