diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java index 22035129..9028fc6d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java @@ -92,6 +92,12 @@ public class RecordAutomationStatusUpdater { for(QRecord record : records) { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // todo - seems like there's some case here, where if an order was in PENDING_INSERT, but then some other job updated the record, that we'd // + // lose that pending status, which would be a Bad Thing™... // + // problem is - we may not have the full record in here, so we can't necessarily check the record to see what status it's currently in... // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + record.setValue(automationDetails.getStatusTracking().getFieldName(), automationStatus.getId()); // todo - another field - for the automation timestamp?? } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/BufferedRecordPipe.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/BufferedRecordPipe.java new file mode 100644 index 00000000..811827c8 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/BufferedRecordPipe.java @@ -0,0 +1,89 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** Subclass of RecordPipe, which uses a buffer in the addRecord method, to avoid + ** sending single-records at a time through postRecordActions and to consumers. + *******************************************************************************/ +public class BufferedRecordPipe extends RecordPipe +{ + private List buffer = new ArrayList<>(); + private Integer bufferSize = 100; + + + + /******************************************************************************* + ** Constructor - uses default buffer size + ** + *******************************************************************************/ + public BufferedRecordPipe() + { + } + + + + /******************************************************************************* + ** Constructor - customize buffer size. + ** + *******************************************************************************/ + public BufferedRecordPipe(Integer bufferSize) + { + this.bufferSize = bufferSize; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void addRecord(QRecord record) + { + buffer.add(record); + if(buffer.size() >= bufferSize) + { + addRecords(buffer); + buffer.clear(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void finalFlush() + { + if(!buffer.isEmpty()) + { + addRecords(buffer); + buffer.clear(); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelExportStreamer.java index 66b6c2ec..6520f6b8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelExportStreamer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelExportStreamer.java @@ -247,14 +247,6 @@ public class ExcelExportStreamer implements ExportStreamerInterface for(QFieldMetaData field : fields) { Serializable value = qRecord.getValue(field.getName()); - if(field.getPossibleValueSourceName() != null) - { - String displayValue = qRecord.getDisplayValue(field.getName()); - if(displayValue != null) - { - value = displayValue; - } - } if(value != null) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java index 21558cd5..98187f79 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java @@ -31,7 +31,7 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; -import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; @@ -43,6 +43,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -147,17 +148,18 @@ public class ExportAction ////////////////////////// // set up a query input // ////////////////////////// - QueryInterface queryInterface = backendModule.getQueryInterface(); - QueryInput queryInput = new QueryInput(exportInput.getInstance()); + QueryAction queryAction = new QueryAction(); + QueryInput queryInput = new QueryInput(exportInput.getInstance()); queryInput.setSession(exportInput.getSession()); queryInput.setTableName(exportInput.getTableName()); queryInput.setFilter(exportInput.getQueryFilter()); queryInput.setLimit(exportInput.getLimit()); + queryInput.setShouldTranslatePossibleValues(true); ///////////////////////////////////////////////////////////////// // tell this query that it needs to put its output into a pipe // ///////////////////////////////////////////////////////////////// - RecordPipe recordPipe = new RecordPipe(); + RecordPipe recordPipe = new BufferedRecordPipe(500); queryInput.setRecordPipe(recordPipe); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -165,13 +167,14 @@ public class ExportAction //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ReportFormat reportFormat = exportInput.getReportFormat(); ExportStreamerInterface reportStreamer = reportFormat.newReportStreamer(); - reportStreamer.start(exportInput, getFields(exportInput), "Sheet 1"); + List fields = getFields(exportInput); + reportStreamer.start(exportInput, fields, "Sheet 1"); ////////////////////////////////////////// // run the query action as an async job // ////////////////////////////////////////// AsyncJobManager asyncJobManager = new AsyncJobManager(); - String queryJobUUID = asyncJobManager.startJob("ReportAction>QueryAction", (status) -> (queryInterface.execute(queryInput))); + String queryJobUUID = asyncJobManager.startJob("ReportAction>QueryAction", (status) -> (queryAction.execute(queryInput))); LOG.info("Started query job [" + queryJobUUID + "] for report"); AsyncJobState queryJobState = AsyncJobState.RUNNING; @@ -209,7 +212,7 @@ public class ExportAction nextSleepMillis = INIT_SLEEP_MS; List records = recordPipe.consumeAvailableRecords(); - reportStreamer.addRecords(records); + processRecords(reportStreamer, fields, records); recordCount += records.size(); LOG.info(countFromPreExecute != null @@ -238,7 +241,7 @@ public class ExportAction // send the final records to the report streamer // /////////////////////////////////////////////////// List records = recordPipe.consumeAvailableRecords(); - reportStreamer.addRecords(records); + processRecords(reportStreamer, fields, records); recordCount += records.size(); long reportEndTime = System.currentTimeMillis(); @@ -269,20 +272,59 @@ public class ExportAction + /******************************************************************************* + ** + *******************************************************************************/ + private static void processRecords(ExportStreamerInterface reportStreamer, List fields, List records) throws QReportingException + { + for(QFieldMetaData field : fields) + { + if(field.getName().endsWith(":possibleValueLabel")) + { + String effectiveFieldName = field.getName().replace(":possibleValueLabel", ""); + for(QRecord record : records) + { + String displayValue = record.getDisplayValue(effectiveFieldName); + record.setValue(field.getName(), displayValue); + } + } + } + + reportStreamer.addRecords(records); + } + + + /******************************************************************************* ** *******************************************************************************/ private List getFields(ExportInput exportInput) { - QTableMetaData table = exportInput.getTable(); + List fieldList; + QTableMetaData table = exportInput.getTable(); if(exportInput.getFieldNames() != null) { - return (exportInput.getFieldNames().stream().map(table::getField).toList()); + fieldList = exportInput.getFieldNames().stream().map(table::getField).toList(); } else { - return (new ArrayList<>(table.getFields().values())); + fieldList = new ArrayList<>(table.getFields().values()); } + + ////////////////////////////////////////// + // add fields for possible value labels // + ////////////////////////////////////////// + List returnList = new ArrayList<>(); + for(QFieldMetaData field : fieldList) + { + returnList.add(field); + if(StringUtils.hasContent(field.getPossibleValueSourceName())) + { + returnList.add(new QFieldMetaData(field.getName() + ":possibleValueLabel", QFieldType.STRING).withLabel(field.getLabel() + " Name")); + } + } + + return (returnList); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java index db426831..e0cd8d21 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java @@ -29,11 +29,13 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.reporting.customizers.DataSourceQueryInputCustomizer; import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportViewCustomizer; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; @@ -68,6 +70,8 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.aggregates.AggregatesInterface; import com.kingsrook.qqq.backend.core.utils.aggregates.BigDecimalAggregates; import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -84,6 +88,8 @@ import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates; *******************************************************************************/ public class GenerateReportAction { + private static final Logger LOG = LogManager.getLogger(GenerateReportAction.class); + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// // summaryAggregates and varianceAggregates are multi-level maps, ala: // // viewName > SummaryKey > fieldName > Aggregates // @@ -214,38 +220,32 @@ public class GenerateReportAction JoinsContext joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins()); - List fields; - if(CollectionUtils.nullSafeHasContents(reportView.getColumns())) + List fields = new ArrayList<>(); + for(QReportField column : reportView.getColumns()) { - fields = new ArrayList<>(); - for(QReportField column : reportView.getColumns()) + if(column.getIsVirtual()) { - if(column.getIsVirtual()) + fields.add(column.toField()); + } + else + { + String effectiveFieldName = Objects.requireNonNullElse(column.getSourceFieldName(), column.getName()); + JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(effectiveFieldName); + if(fieldAndTableNameOrAlias.field() == null) { - fields.add(column.toField()); + throw new QReportingException("Could not find field named [" + effectiveFieldName + "] on table [" + table.getName() + "]"); } - else - { - JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(column.getName()); - if(fieldAndTableNameOrAlias.field() == null) - { - throw new QReportingException("Could not find field named [" + column.getName() + "] on table [" + table.getName() + "]"); - } - QFieldMetaData field = fieldAndTableNameOrAlias.field().clone(); - field.setName(column.getName()); - if(StringUtils.hasContent(column.getLabel())) - { - field.setLabel(column.getLabel()); - } - fields.add(field); + QFieldMetaData field = fieldAndTableNameOrAlias.field().clone(); + field.setName(column.getName()); + if(StringUtils.hasContent(column.getLabel())) + { + field.setLabel(column.getLabel()); } + fields.add(field); } } - else - { - fields = new ArrayList<>(table.getFields().values()); - } + reportStreamer.setDisplayFormats(getDisplayFormatMap(fields)); reportStreamer.start(exportInput, fields, reportView.getLabel()); } @@ -286,7 +286,7 @@ public class GenerateReportAction ///////////////////////////////////////////////////////////////// // run a record pipe loop, over the query for this data source // ///////////////////////////////////////////////////////////////// - RecordPipe recordPipe = new RecordPipe(); + RecordPipe recordPipe = new BufferedRecordPipe(1000); new AsyncRecordPipeLoop().run("Report[" + reportInput.getReportName() + "]", null, recordPipe, (callback) -> { if(dataSource.getSourceTable() != null) @@ -301,6 +301,13 @@ public class GenerateReportAction queryInput.setFilter(queryFilter); queryInput.setQueryJoins(dataSource.getQueryJoins()); queryInput.setShouldTranslatePossibleValues(true); // todo - any limits or conditions on this? + + if(dataSource.getQueryInputCustomizer() != null) + { + DataSourceQueryInputCustomizer queryInputCustomizer = QCodeLoader.getAdHoc(DataSourceQueryInputCustomizer.class, dataSource.getQueryInputCustomizer()); + queryInput = queryInputCustomizer.run(reportInput, queryInput); + } + return (new QueryAction().execute(queryInput)); } else if(dataSource.getStaticDataSupplier() != null) @@ -368,8 +375,9 @@ public class GenerateReportAction for(Serializable value : criterion.getValues()) { - String valueAsString = ValueUtils.getValueAsString(value); - Serializable interpretedValue = variableInterpreter.interpret(valueAsString); + String valueAsString = ValueUtils.getValueAsString(value); + // Serializable interpretedValue = variableInterpreter.interpret(valueAsString); + Serializable interpretedValue = variableInterpreter.interpretForObject(valueAsString); newValues.add(interpretedValue); } criterion.setValues(newValues); @@ -391,6 +399,22 @@ public class GenerateReportAction //////////////////////////////////////////////////////////////////////////// if(tableView != null) { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if any fields are 'showPossibleValueLabel', then move display values for them into the record's values map // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QReportField column : tableView.getColumns()) + { + if(column.getShowPossibleValueLabel()) + { + String effectiveFieldName = Objects.requireNonNullElse(column.getSourceFieldName(), column.getName()); + for(QRecord record : records) + { + String displayValue = record.getDisplayValue(effectiveFieldName); + record.setValue(column.getName(), displayValue); + } + } + } + reportStreamer.addRecords(records); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java index c58f2930..90d76f2d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java @@ -24,14 +24,18 @@ package com.kingsrook.qqq.backend.core.actions.reporting; import java.io.IOException; import java.io.OutputStream; +import java.io.Serializable; import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -49,6 +53,7 @@ public class JsonExportStreamer implements ExportStreamerInterface private OutputStream outputStream; private boolean needComma = false; + private boolean prettyPrint = true; @@ -74,7 +79,7 @@ public class JsonExportStreamer implements ExportStreamerInterface try { - outputStream.write("[".getBytes(StandardCharsets.UTF_8)); + outputStream.write('['); } catch(IOException e) { @@ -109,11 +114,25 @@ public class JsonExportStreamer implements ExportStreamerInterface { if(needComma) { - outputStream.write(",".getBytes(StandardCharsets.UTF_8)); + outputStream.write(','); + } + + Map mapForJson = new LinkedHashMap<>(); + for(QFieldMetaData field : fields) + { + String labelForJson = StringUtils.lcFirst(field.getLabel().replace(" ", "")); + mapForJson.put(labelForJson, qRecord.getValue(field.getName())); + } + + String json = prettyPrint ? JsonUtils.toPrettyJson(mapForJson) : JsonUtils.toJson(mapForJson); + + if(prettyPrint) + { + outputStream.write('\n'); } - String json = JsonUtils.toJson(qRecord); outputStream.write(json.getBytes(StandardCharsets.UTF_8)); + outputStream.flush(); // todo - less often? needComma = true; } @@ -144,7 +163,11 @@ public class JsonExportStreamer implements ExportStreamerInterface { try { - outputStream.write("]".getBytes(StandardCharsets.UTF_8)); + if(prettyPrint) + { + outputStream.write('\n'); + } + outputStream.write(']'); } catch(IOException e) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/customizers/DataSourceQueryInputCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/customizers/DataSourceQueryInputCustomizer.java new file mode 100644 index 00000000..a20c7997 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/customizers/DataSourceQueryInputCustomizer.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting.customizers; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; + + +/******************************************************************************* + ** Interface for customizer on a QReportDataSource's query. + ** + ** Useful, for example, to look at what input field values were given, and change + ** the query filter (e.g., conditional criteria), or issue an error based on the + ** combination of input fields given. + *******************************************************************************/ +public interface DataSourceQueryInputCustomizer +{ + + /******************************************************************************* + ** + *******************************************************************************/ + QueryInput run(ReportInput reportInput, QueryInput queryInput) throws QException; + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java index e757a4ae..a63de9b9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java @@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.reporting.BufferedRecordPipe; import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -36,6 +37,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -44,6 +47,8 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; *******************************************************************************/ public class QueryAction { + private static final Logger LOG = LogManager.getLogger(QueryAction.class); + private Optional postQueryRecordCustomizer; private QueryInput queryInput; @@ -72,6 +77,11 @@ public class QueryAction QueryOutput queryOutput = qModule.getQueryInterface().execute(queryInput); // todo post-customization - can do whatever w/ the result if you want + if(queryInput.getRecordPipe() instanceof BufferedRecordPipe bufferedRecordPipe) + { + bufferedRecordPipe.finalFlush(); + } + if(queryInput.getRecordPipe() == null) { postRecordActions(queryOutput.getRecords()); @@ -100,7 +110,7 @@ public class QueryAction { qPossibleValueTranslator = new QPossibleValueTranslator(queryInput.getInstance(), queryInput.getSession()); } - qPossibleValueTranslator.translatePossibleValuesInRecords(queryInput.getTable(), records); + qPossibleValueTranslator.translatePossibleValuesInRecords(queryInput.getTable(), records, queryInput.getQueryJoins()); } if(queryInput.getShouldGenerateDisplayValues()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java index 72730b05..b2728052 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.actions.values; import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -38,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperat import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -50,6 +52,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -91,13 +95,24 @@ public class QPossibleValueTranslator ** For a list of records, translate their possible values (populating their display values) *******************************************************************************/ public void translatePossibleValuesInRecords(QTableMetaData table, List records) + { + translatePossibleValuesInRecords(table, records, Collections.emptyList()); + } + + + + /******************************************************************************* + ** For a list of records, translate their possible values (populating their display values) + *******************************************************************************/ + public void translatePossibleValuesInRecords(QTableMetaData table, List records, List queryJoins) { if(records == null || table == null) { return; } - primePvsCache(table, records); + LOG.debug("Translating possible values in [" + records.size() + "] records from the [" + table.getName() + "] table."); + primePvsCache(table, records, queryJoins); for(QRecord record : records) { @@ -108,6 +123,42 @@ public class QPossibleValueTranslator record.setDisplayValue(field.getName(), translatePossibleValue(field, record.getValue(field.getName()))); } } + + for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryJoins)) + { + if(queryJoin.getSelect()) + { + try + { + //////////////////////////////////////////// + // todo - aliases aren't be handled right // + //////////////////////////////////////////// + QTableMetaData joinTable = qInstance.getTable(queryJoin.getRightTable()); + for(QFieldMetaData field : joinTable.getFields().values()) + { + if(field.getPossibleValueSourceName() != null) + { + /////////////////////////////////////////////// + // avoid circling-back upon the source table // + /////////////////////////////////////////////// + QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(field.getPossibleValueSourceName()); + if(QPossibleValueSourceType.TABLE.equals(possibleValueSource.getType()) && table.getName().equals(possibleValueSource.getTableName())) + { + continue; + } + + String joinFieldName = joinTable.getName() + "." + field.getName(); + record.setDisplayValue(joinFieldName, translatePossibleValue(field, record.getValue(joinFieldName))); + } + } + } + catch(Exception e) + { + LOG.warn("Error translating join table possible values", e); + } + } + } + } } @@ -244,8 +295,7 @@ public class QPossibleValueTranslator ////////////////////////////////////////////////////////////// // look for cached value - if it's missing, call the primer // ////////////////////////////////////////////////////////////// - possibleValueCache.putIfAbsent(possibleValueSource.getName(), new HashMap<>()); - Map cacheForPvs = possibleValueCache.get(possibleValueSource.getName()); + Map cacheForPvs = possibleValueCache.computeIfAbsent(possibleValueSource.getName(), x -> new HashMap<>()); if(!cacheForPvs.containsKey(value)) { primePvsCache(possibleValueSource.getTableName(), List.of(possibleValueSource), List.of(value)); @@ -329,21 +379,29 @@ public class QPossibleValueTranslator /******************************************************************************* ** prime the cache (e.g., by doing bulk-queries) for table-based PVS's - ** - ** @param table the table that the records are from + ** @param table the table that the records are from ** @param records the records that have the possible value id's (e.g., foreign keys) + * @param queryJoins *******************************************************************************/ - void primePvsCache(QTableMetaData table, List records) + void primePvsCache(QTableMetaData table, List records, List queryJoins) { - ListingHash fieldsByPvsTable = new ListingHash<>(); - ListingHash pvsesByTable = new ListingHash<>(); - for(QFieldMetaData field : table.getFields().values()) + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // this is a map of String(tableName - the PVS table) to Pair(String (either "" for main table in a query, or join-table + "."), field (from the table being selected from)) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ListingHash> fieldsByPvsTable = new ListingHash<>(); + + /////////////////////////////////////////////////////////////////////////////////////// + // this is a map of String(tableName - the PVS table) to PossibleValueSource objects // + /////////////////////////////////////////////////////////////////////////////////////// + ListingHash pvsesByTable = new ListingHash<>(); + + primePvsCacheTableListingHashLoader(table, fieldsByPvsTable, pvsesByTable, ""); + for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryJoins)) { - QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(field.getPossibleValueSourceName()); - if(possibleValueSource != null && possibleValueSource.getType().equals(QPossibleValueSourceType.TABLE)) + if(queryJoin.getSelect()) { - fieldsByPvsTable.add(possibleValueSource.getTableName(), field); - pvsesByTable.add(possibleValueSource.getTableName(), possibleValueSource); + // todo - aliases probably not handled right + primePvsCacheTableListingHashLoader(qInstance.getTable(queryJoin.getRightTable()), fieldsByPvsTable, pvsesByTable, queryJoin.getRightTable() + "."); } } @@ -352,16 +410,24 @@ public class QPossibleValueTranslator Set values = new HashSet<>(); for(QRecord record : records) { - for(QFieldMetaData field : fieldsByPvsTable.get(tableName)) + for(Pair fieldPair : fieldsByPvsTable.get(tableName)) { - Serializable fieldValue = record.getValue(field.getName()); + String fieldName = fieldPair.getA() + fieldPair.getB().getName(); + Serializable fieldValue = record.getValue(fieldName); + + ///////////////////////////////////////// + // ignore null and empty-string values // + ///////////////////////////////////////// + if(!StringUtils.hasContent(ValueUtils.getValueAsString(fieldValue))) + { + continue; + } ////////////////////////////////////// // check if value is already cached // ////////////////////////////////////// QPossibleValueSource possibleValueSource = pvsesByTable.get(tableName).get(0); - possibleValueCache.putIfAbsent(possibleValueSource.getName(), new HashMap<>()); - Map cacheForPvs = possibleValueCache.get(possibleValueSource.getName()); + Map cacheForPvs = possibleValueCache.computeIfAbsent(possibleValueSource.getName(), x -> new HashMap<>()); if(!cacheForPvs.containsKey(fieldValue)) { @@ -379,6 +445,28 @@ public class QPossibleValueTranslator + /******************************************************************************* + ** Helper for the primePvsCache method + *******************************************************************************/ + private void primePvsCacheTableListingHashLoader(QTableMetaData table, ListingHash> fieldsByPvsTable, ListingHash pvsesByTable, String fieldNamePrefix) + { + for(QFieldMetaData field : table.getFields().values()) + { + QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(field.getPossibleValueSourceName()); + if(possibleValueSource != null && possibleValueSource.getType().equals(QPossibleValueSourceType.TABLE)) + { + fieldsByPvsTable.add(possibleValueSource.getTableName(), Pair.of(fieldNamePrefix, field)); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // todo - optimization we can put the same PVS in this listing hash multiple times... either check for dupes, or change to a set, or something smarter. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + pvsesByTable.add(possibleValueSource.getTableName(), possibleValueSource); + } + } + } + + + /******************************************************************************* ** For a given table, and a list of pkey-values in that table, AND a list of ** possible value sources based on that table (maybe usually 1, but could be more, @@ -408,10 +496,12 @@ public class QPossibleValueTranslator ///////////////////////////////////////////////////////////////////////////////////////// if(notTooDeep()) { - queryInput.setShouldTranslatePossibleValues(true); + // todo not commit... + // queryInput.setShouldTranslatePossibleValues(true); queryInput.setShouldGenerateDisplayValues(true); } + LOG.debug("Priming PVS cache for [" + page.size() + "] ids from [" + tableName + "] table."); QueryOutput queryOutput = new QueryAction().execute(queryInput); /////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 4e2271d0..b77c2519 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; @@ -58,6 +59,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; @@ -984,7 +986,7 @@ public class QInstanceValidator { if(dataSource.getQueryFilter() != null) { - validateQueryFilter("In " + dataSourceErrorPrefix + "query filter - ", qInstance.getTable(dataSource.getSourceTable()), dataSource.getQueryFilter()); + validateQueryFilter(qInstance, "In " + dataSourceErrorPrefix + "query filter - ", qInstance.getTable(dataSource.getSourceTable()), dataSource.getQueryFilter(), dataSource.getQueryJoins()); } } } @@ -999,9 +1001,9 @@ public class QInstanceValidator } } - //////////////////////////////////////// - // validate dataSources in the report // - //////////////////////////////////////// + ////////////////////////////////// + // validate views in the report // + ////////////////////////////////// if(assertCondition(CollectionUtils.nullSafeHasContents(report.getViews()), "At least 1 view must be defined in report " + reportName + ".")) { int index = 0; @@ -1015,19 +1017,30 @@ public class QInstanceValidator usedViewNames.add(view.getName()); String viewErrorPrefix = "Report " + reportName + " view " + view.getName() + " "; - assertCondition(view.getType() != null, viewErrorPrefix + " is missing its type."); - if(assertCondition(StringUtils.hasContent(view.getDataSourceName()), viewErrorPrefix + " is missing a dataSourceName")) + assertCondition(view.getType() != null, viewErrorPrefix + "is missing its type."); + if(assertCondition(StringUtils.hasContent(view.getDataSourceName()), viewErrorPrefix + "is missing a dataSourceName")) { - assertCondition(usedDataSourceNames.contains(view.getDataSourceName()), viewErrorPrefix + " has an unrecognized dataSourceName: " + view.getDataSourceName()); + assertCondition(usedDataSourceNames.contains(view.getDataSourceName()), viewErrorPrefix + "has an unrecognized dataSourceName: " + view.getDataSourceName()); } if(StringUtils.hasContent(view.getVarianceDataSourceName())) { - assertCondition(usedDataSourceNames.contains(view.getVarianceDataSourceName()), viewErrorPrefix + " has an unrecognized varianceDataSourceName: " + view.getVarianceDataSourceName()); + assertCondition(usedDataSourceNames.contains(view.getVarianceDataSourceName()), viewErrorPrefix + "has an unrecognized varianceDataSourceName: " + view.getVarianceDataSourceName()); } - // actually, this is okay if there's a customizer, so... - assertCondition(CollectionUtils.nullSafeHasContents(view.getColumns()), viewErrorPrefix + " does not have any columns."); + boolean hasColumns = CollectionUtils.nullSafeHasContents(view.getColumns()); + boolean hasViewCustomizer = view.getViewCustomizer() != null; + assertCondition(hasColumns || hasViewCustomizer, viewErrorPrefix + "does not have any columns or a view customizer."); + + Set usedColumnNames = new HashSet<>(); + for(QReportField column : CollectionUtils.nonNullList(view.getColumns())) + { + assertCondition(StringUtils.hasContent(column.getName()), viewErrorPrefix + "has a column with no name."); + assertCondition(!usedColumnNames.contains(column.getName()), viewErrorPrefix + "has multiple columns named: " + column.getName()); + usedColumnNames.add(column.getName()); + + // todo - is field name valid? + } // todo - all these too... // view.getPivotFields(); @@ -1047,34 +1060,74 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateQueryFilter(String context, QTableMetaData table, QQueryFilter queryFilter) + private void validateQueryFilter(QInstance qInstance, String context, QTableMetaData table, QQueryFilter queryFilter, List queryJoins) { for(QFilterCriteria criterion : CollectionUtils.nonNullList(queryFilter.getCriteria())) { - if(assertCondition(StringUtils.hasContent(criterion.getFieldName()), context + "Missing fieldName for a criteria")) + String fieldName = criterion.getFieldName(); + if(assertCondition(StringUtils.hasContent(fieldName), context + "Missing fieldName for a criteria")) { - assertNoException(() -> table.getField(criterion.getFieldName()), context + "Criteria fieldName " + criterion.getFieldName() + " is not a field in this table."); + assertCondition(findField(qInstance, table, queryJoins, fieldName), context + "Criteria fieldName " + fieldName + " is not a field in this table (or in any given joins)."); } - assertCondition(criterion.getOperator() != null, context + "Missing operator for a criteria on fieldName " + criterion.getFieldName()); - assertCondition(criterion.getValues() != null, context + "Missing values for a criteria on fieldName " + criterion.getFieldName()); // todo - what about ops w/ no value (BLANK) + assertCondition(criterion.getOperator() != null, context + "Missing operator for a criteria on fieldName " + fieldName); + assertCondition(criterion.getValues() != null, context + "Missing values for a criteria on fieldName " + fieldName); // todo - what about ops w/ no value (BLANK) } for(QFilterOrderBy orderBy : CollectionUtils.nonNullList(queryFilter.getOrderBys())) { if(assertCondition(StringUtils.hasContent(orderBy.getFieldName()), context + "Missing fieldName for an orderBy")) { - assertNoException(() -> table.getField(orderBy.getFieldName()), context + "OrderBy fieldName " + orderBy.getFieldName() + " is not a field in this table."); + assertCondition(findField(qInstance, table, queryJoins, orderBy.getFieldName()), context + "OrderBy fieldName " + orderBy.getFieldName() + " is not a field in this table (or in any given joins)."); } } for(QQueryFilter subFilter : CollectionUtils.nonNullList(queryFilter.getSubFilters())) { - validateQueryFilter(context, table, subFilter); + validateQueryFilter(qInstance, context, table, subFilter, queryJoins); } } + /******************************************************************************* + ** Look for a field name in either a table, or the tables referenced in a list of query joins. + *******************************************************************************/ + private static boolean findField(QInstance qInstance, QTableMetaData table, List queryJoins, String fieldName) + { + boolean foundField = false; + try + { + table.getField(fieldName); + foundField = true; + } + catch(Exception e) + { + if(fieldName.contains(".")) + { + String fieldNameAfterDot = fieldName.substring(fieldName.lastIndexOf(".") + 1); + for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryJoins)) + { + QTableMetaData joinTable = qInstance.getTable(queryJoin.getRightTable()); + if(joinTable != null) + { + try + { + joinTable.getField(fieldNameAfterDot); + foundField = true; + } + catch(Exception e2) + { + continue; + } + } + } + } + } + return foundField; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportDataSource.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportDataSource.java index 260e51bb..2473e921 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportDataSource.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportDataSource.java @@ -42,6 +42,7 @@ public class QReportDataSource private List queryJoins = null; + private QCodeReference queryInputCustomizer; private QCodeReference staticDataSupplier; @@ -230,4 +231,38 @@ public class QReportDataSource return (this); } + + + /******************************************************************************* + ** Getter for queryInputCustomizer + ** + *******************************************************************************/ + public QCodeReference getQueryInputCustomizer() + { + return queryInputCustomizer; + } + + + + /******************************************************************************* + ** Setter for queryInputCustomizer + ** + *******************************************************************************/ + public void setQueryInputCustomizer(QCodeReference queryInputCustomizer) + { + this.queryInputCustomizer = queryInputCustomizer; + } + + + + /******************************************************************************* + ** Fluent setter for queryInputCustomizer + ** + *******************************************************************************/ + public QReportDataSource withQueryInputCustomizer(QCodeReference queryInputCustomizer) + { + this.queryInputCustomizer = queryInputCustomizer; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportField.java index ab8885c7..e588b511 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportField.java @@ -43,6 +43,9 @@ public class QReportField private boolean isVirtual = false; + private boolean showPossibleValueLabel = false; + private String sourceFieldName; + /******************************************************************************* @@ -281,4 +284,73 @@ public class QReportField this.isVirtual = isVirtual; return (this); } + + + + /******************************************************************************* + ** Getter for sourceFieldName + ** + *******************************************************************************/ + public String getSourceFieldName() + { + return sourceFieldName; + } + + + + /******************************************************************************* + ** Setter for sourceFieldName + ** + *******************************************************************************/ + public void setSourceFieldName(String sourceFieldName) + { + this.sourceFieldName = sourceFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for sourceFieldName + ** + *******************************************************************************/ + public QReportField withSourceFieldName(String sourceFieldName) + { + this.sourceFieldName = sourceFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for showPossibleValueLabel + ** + *******************************************************************************/ + public boolean getShowPossibleValueLabel() + { + return showPossibleValueLabel; + } + + + + /******************************************************************************* + ** Setter for showPossibleValueLabel + ** + *******************************************************************************/ + public void setShowPossibleValueLabel(boolean showPossibleValueLabel) + { + this.showPossibleValueLabel = showPossibleValueLabel; + } + + + + /******************************************************************************* + ** Fluent setter for showPossibleValueLabel + ** + *******************************************************************************/ + public QReportField withShowPossibleValueLabel(boolean showPossibleValueLabel) + { + this.showPossibleValueLabel = showPossibleValueLabel; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java index 2ee8e59e..0fc8115d 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java @@ -241,8 +241,8 @@ public class StringUtils return (null); } - StringBuilder rs = new StringBuilder(); - int size = input.size(); + StringBuilder rs = new StringBuilder(); + int size = input.size(); for(int i = 0; i < size; i++) { @@ -361,4 +361,44 @@ public class StringUtils return (size != null && size.equals(1) ? ifOne : ifNotOne); } + + + /******************************************************************************* + ** Lowercase the first char of a string. + *******************************************************************************/ + public static String lcFirst(String s) + { + if(s == null) + { + return (null); + } + + if(s.length() <= 1) + { + return (s.toLowerCase()); + } + + return (s.substring(0, 1).toLowerCase() + s.substring(1)); + } + + + + /******************************************************************************* + ** Uppercase the first char of a string. + *******************************************************************************/ + public static String ucFirst(String s) + { + if(s == null) + { + return (null); + } + + if(s.length() <= 1) + { + return (s.toUpperCase()); + } + + return (s.substring(0, 1).toUpperCase() + s.substring(1)); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java index ba66dcbb..d1f98e40 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslatorTest.java @@ -188,7 +188,7 @@ public class QPossibleValueTranslatorTest ); QTableMetaData personTable = qInstance.getTable(TestUtils.TABLE_NAME_PERSON); MemoryRecordStore.resetStatistics(); - possibleValueTranslator.primePvsCache(personTable, personRecords); + possibleValueTranslator.primePvsCache(personTable, personRecords, null); // todo - test non-null queryJoins assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should only run 1 query"); possibleValueTranslator.translatePossibleValue(shapeField, 1); possibleValueTranslator.translatePossibleValue(shapeField, 2); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 14e1d2c8..83a1f3f3 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -55,6 +55,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -1436,6 +1437,69 @@ class QInstanceValidatorTest dataSource.setStaticDataSupplier(new QCodeReference(ArrayList.class, null)); }, "is not of the expected type"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testReportViewBasics() + { + assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).setViews(null), + "At least 1 view must be defined in report"); + + assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).setViews(new ArrayList<>()), + "At least 1 view must be defined in report"); + + ///////////////////////////////////////////////////////////////////////// + // meh, enricher sets a default name, so, can't easily catch this one. // + ///////////////////////////////////////////////////////////////////////// + // assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getViews().get(0).setName(null), + // "Missing name for a view"); + // assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getViews().get(0).setName(""), + // "Missing name for a view"); + + assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getViews().get(0).setType(null), + "missing its type"); + + ///////////////////////////////////////////////////////////////////////// + // meh, enricher sets a default name, so, can't easily catch this one. // + ///////////////////////////////////////////////////////////////////////// + // assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getViews().get(0).setDataSourceName(null), + // "missing a dataSourceName"); + // assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getViews().get(0).setDataSourceName(""), + // "missing a dataSourceName"); + + assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getViews().get(0).setDataSourceName("notADataSource"), + "has an unrecognized dataSourceName"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testReportViewColumns() + { + assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getViews().get(0).setColumns(null), + "does not have any columns or a view customizer"); + + assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getViews().get(0).getColumns().get(0).setName(null), + "has a column with no name"); + + assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getViews().get(0).getColumns().get(0).setName(""), + "has a column with no name"); + + assertValidationFailureReasons((qInstance) -> + { + List columns = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getViews().get(0).getColumns(); + columns.get(0).setName("id"); + columns.get(1).setName("id"); + }, + "has multiple columns named: id"); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java index 59d63d0d..3e75eb8d 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java @@ -182,6 +182,7 @@ class StringUtilsTest } + /******************************************************************************* ** *******************************************************************************/ @@ -246,4 +247,41 @@ class StringUtilsTest assertEquals("", StringUtils.plural(1, "", "es")); assertEquals("es", StringUtils.plural(2, "", "es")); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testLcFirst() + { + assertNull(StringUtils.lcFirst(null)); + assertEquals("", StringUtils.lcFirst("")); + assertEquals(" ", StringUtils.lcFirst(" ")); + assertEquals("a", StringUtils.lcFirst("A")); + assertEquals("1", StringUtils.lcFirst("1")); + assertEquals("a", StringUtils.lcFirst("a")); + assertEquals("aB", StringUtils.lcFirst("AB")); + assertEquals("aBc", StringUtils.lcFirst("ABc")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testUcFirst() + { + assertNull(StringUtils.ucFirst(null)); + assertEquals("", StringUtils.ucFirst("")); + assertEquals(" ", StringUtils.ucFirst(" ")); + assertEquals("A", StringUtils.ucFirst("A")); + assertEquals("1", StringUtils.ucFirst("1")); + assertEquals("A", StringUtils.ucFirst("a")); + assertEquals("Ab", StringUtils.ucFirst("ab")); + assertEquals("Abc", StringUtils.ucFirst("abc")); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 2a7db9bb..347d1417 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -47,6 +47,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput; @@ -140,6 +141,7 @@ public class TestUtils public static final String TABLE_NAME_ID_AND_NAME_ONLY = "idAndNameOnly"; public static final String TABLE_NAME_BASEPULL = "basepullTest"; public static final String REPORT_NAME_SHAPES_PERSON = "shapesPersonReport"; + public static final String REPORT_NAME_PERSON_JOIN_SHAPE = "simplePersonReport"; public static final String POSSIBLE_VALUE_SOURCE_STATE = "state"; // enum-type public static final String POSSIBLE_VALUE_SOURCE_SHAPE = "shape"; // table-type @@ -192,6 +194,7 @@ public class TestUtils qInstance.addReport(defineShapesPersonsReport()); qInstance.addProcess(defineShapesPersonReportProcess()); + qInstance.addReport(definePersonJoinShapeReport()); qInstance.addAutomationProvider(definePollingAutomationProvider()); @@ -1108,4 +1111,26 @@ public class TestUtils .getProcessMetaData(); } + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QReportMetaData definePersonJoinShapeReport() + { + return new QReportMetaData() + .withName(REPORT_NAME_PERSON_JOIN_SHAPE) + .withDataSource( + new QReportDataSource() + .withSourceTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + ) + .withView(new QReportView() + .withType(ReportType.TABLE) + .withColumns(List.of( + new QReportField("id"), + new QReportField("firstName"), + new QReportField("lastName") + )) + ); + } } diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 92acbc89..f4783a6a 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -926,8 +926,7 @@ public class QJavalinImplementation } catch(Exception e) { - pipedOutputStream.write(("Error generating report: " + e.getMessage()).getBytes()); - pipedOutputStream.close(); + handleExportOrReportException(context, pipedOutputStream, e); return (false); } }); @@ -953,6 +952,41 @@ public class QJavalinImplementation + /******************************************************************************* + ** + *******************************************************************************/ + private static void handleExportOrReportException(Context context, PipedOutputStream pipedOutputStream, Exception e) throws IOException + { + HttpStatus.Code statusCode = HttpStatus.Code.INTERNAL_SERVER_ERROR; // 500 + String message = e.getMessage(); + + QUserFacingException userFacingException = ExceptionUtils.findClassInRootChain(e, QUserFacingException.class); + if(userFacingException != null) + { + LOG.info("User-facing exception", e); + statusCode = HttpStatus.Code.BAD_REQUEST; // 400 + message = userFacingException.getMessage(); + } + else + { + QAuthenticationException authenticationException = ExceptionUtils.findClassInRootChain(e, QAuthenticationException.class); + if(authenticationException != null) + { + statusCode = HttpStatus.Code.UNAUTHORIZED; // 401 + } + else + { + LOG.warn("Unexpected exception in javalin report or export request", e); + } + } + + context.status(statusCode.getCode()); + pipedOutputStream.write(("Error generating report: " + message).getBytes()); + pipedOutputStream.close(); + } + + + /******************************************************************************* ** *******************************************************************************/